Refactor data generators into seperate classes

Use callback pattern to handle processing of individual rows of data from data generators
Use dataclasses to define individual data instances

gitstats/__init__.py

@@ -1,11 +1,7 @@
-from gitstats.gitstats import GitStats
+from gitstats.process import run
 from gitstats._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
 
-def main():
-    g = GitStats()
-    g.run()
-
 if __name__ == "__main__":
-    main()
+    run()

gitstats/cd.py

@@ -0,0 +1,16 @@
+"""Holds the current working directory context class. A python version of pushd/popd"""
+import os
+
+# pylint: disable=too-few-public-methods
+class cd: # pylint: disable=invalid-name
+    """Context manager for changing the current working directory"""
+    def __init__(self, newPath):
+        self.new_path = os.path.expanduser(newPath)
+        self.saved_path = os.getcwd()
+
+    def __enter__(self):
+        self.saved_path = os.getcwd()
+        os.chdir(self.new_path)
+
+    def __exit__(self, etype, value, traceback):
+        os.chdir(self.saved_path)

gitstats/cli.py

@@ -0,0 +1,60 @@
+import getopt
+import logging
+import os
+import sys
+
+conf = {
+    'max_domains': 10,
+    'max_ext_length': 10,
+    'style': 'gitstats.css',
+    'max_authors': 20,
+    'authors_top': 5,
+    'commit_begin': '',
+    'commit_end': 'HEAD',
+    'linear_linestats': 1,
+    'project_name': '',
+    'processes': 8,
+    'start_date': '',
+    'logging': logging.INFO,
+}
+
+
+def _usage():
+    print(f"""
+Usage: gitstats [options] <gitpath..> <outputpath>
+
+Options:
+-c key=value     Override configuration value
+-n key=value     Define author name equivalency (key will treated the same as value)
+
+Default config values:
+{conf}
+
+Please see the manual page for more details.
+""")
+
+
+def get_cli():
+    optlist, args = getopt.getopt(sys.argv[1:], 'hc:', ["help"])
+    for o, v in optlist:
+        if o == '-c':
+            key, value = v.split('=', 1)
+            if key not in conf:
+                raise KeyError('no such key "%s" in config' % key)
+            if isinstance(conf[key], int):
+                conf[key] = int(value)
+            else:
+                conf[key] = value
+        elif o in ('-h', '--help'):
+            _usage()
+            sys.exit()
+
+    if len(args) < 2:
+        _usage()
+        sys.exit(0)
+
+    outputpath = os.path.abspath(args[-1])
+    paths = args[0:-1]
+    outputpath = os.path.abspath(outputpath)
+
+    return conf, paths, outputpath

gitstats/data/__init__.py

@@ -0,0 +1,7 @@
+from .author import Author
+from .author_row import AuthorRow
+from .author_totals import AuthorTotals
+from .tag import Tag
+from .revision import Revision
+from .file import File
+from .loc_by_date import LocByDate

gitstats/data/author_row.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass
+class AuthorRow:
+    sha: str
+    stamp: int
+    author: str
+    files_modified: int
+    lines_inserted: int
+    lines_deleted: int

gitstats/data/author_totals.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+@dataclass
+class AuthorTotals:
+    author: str
+    total_commits: int

gitstats/data/file.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass
+class File:
+    full_path: str
+    ext: str
+    size: int
+    lines: int = 0
+
+

gitstats/data/loc_by_date.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class LocByDate:
+    hash: str = ''
+    stamp: int = 0
+    file_count: int = 0
+    lines_inserted: int = 0
+    lines_deleted: int = 0
+    total_lines: int = 0
+
+

gitstats/data/revision.py

@@ -0,0 +1,16 @@
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Dict
+
+#    # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
+
+@dataclass
+class Revision:
+    sha: str
+    stamp: int
+    timezone: int = 0
+    author: str = ''
+    email: str = ''
+    domain: str = ''
+    file_count: int = 0
+

gitstats/data/tag.py

@@ -0,0 +1,13 @@
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Dict
+
+@dataclass
+class Tag:
+    tag: str
+    stamp: int
+    hash: str
+    commits: int = 0
+    authors: Dict[str, int] = field(default_factory=defaultdict(int))
+
+

gitstats/data_generators/__init__.py

@@ -0,0 +1,6 @@
+from .gen_author_data import gen_author_data
+from .gen_author_totals import gen_author_totals_data
+from .gen_tag_data import gen_tag_data
+from .gen_revision_data import gen_revision_data
+from .gen_file_data import gen_file_data
+from .gen_loc_data import gen_loc_data

gitstats/data_generators/gen_author_data.py

@@ -0,0 +1,88 @@
+import csv
+import logging
+import os
+import re
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getlogrange, getpipeoutput, getstatsummarycounts
+from gitstats.data import AuthorRow
+
+
+def gen_author_data(conf, row_processor):
+    '''
+    Given a configuration, pull authorship information. For
+    each author, callback to the row_processor passing an AuthorRow
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: None
+    '''
+
+    # DBG: git log --shortstat --date-order --pretty=format:"%H %at %aN" --since="2017-10-01" "HEAD"
+    # Results are in the form of
+    #
+    # 3c16756701d264619db0b309f42ebdc713b29827 1522513256 Dan Rapp
+    # 524ee0d32ffbbb8bb82966b769bbf7dbc1d87a68 1522480979 Michael Wright
+    # 1 file changed, 6 insertions(+)
+    #
+    # If there are two (or more) lines,
+    # The first line(s) is the merge to master or other branch
+    # The last line is the commit on the branch
+    lines = getpipeoutput(
+        ['git log --shortstat --date-order --pretty=format:"%%H %%at %%aN" %s' % (
+            getlogrange(conf, 'HEAD'))]).split('\n')
+    lines.reverse()
+
+    files = 0
+    inserted = 0
+    deleted = 0
+    stamp = 0
+    for line in lines:
+        if len(line) == 0:
+            continue
+
+        # <stamp> <author>
+        if re.search('files? changed', line) is None:
+            if files + inserted + deleted > 0:  # this case indicates we've already processed the line
+                pos = line.find(' ')
+                if pos != -1:
+                    try:
+                        oldstamp = stamp
+                        tokens = line.split()
+                        sha = tokens[0]
+                        stamp = int(tokens[1])
+                        author = ' '.join(tokens[2:])
+                        if oldstamp > stamp:
+                            # clock skew, keep old timestamp to avoid having ugly graph
+                            stamp = oldstamp
+                        row_processor(AuthorRow(sha, stamp, author, files, inserted, deleted))
+                        # Since subsequent lines are (generally) reflections of merging into a branch
+                        # don't provide "credit" to the author did the merge
+                        (files, inserted, deleted) = 0, 0, 0
+                    except ValueError:
+                        logging.warning(f'unexpected line "{line}')
+                else:
+                    logging.warning(f'unexpected line "{line}')
+        else:
+            numbers = getstatsummarycounts(line)
+
+            if len(numbers) == 3:
+                (files, inserted, deleted) = map(lambda el: int(el), numbers)
+            else:
+                logging.warning(f'Failed to handle line "{line}"')
+                (files, inserted, deleted) = (0, 0, 0)
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'sha', 'stamp', 'author', 'files changed', 'lines inserted', 'lines deleted'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+
+                gen_author_data(
+                    conf,
+                    lambda row: writer.writerow([repo_name, row.sha, row.stamp, row.author, row.files_modified,
+                                                 row.lines_inserted, row.lines_deleted]))

gitstats/data_generators/gen_author_totals.py

@@ -0,0 +1,47 @@
+import csv
+import os
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getlogrange, getpipeoutput
+from gitstats.data import AuthorTotals
+
+
+def gen_author_totals_data(conf, row_processor=None, revision_range=None):
+    '''
+    Given configuration, pull total commit per author. For
+    each "row" callback to the row_processor passing an AuthorTotals
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: count of the number of authors
+    '''
+
+    # DBG: git shortlog -s --since="2017-10-01" "HEAD"
+    if not revision_range:
+        revision_range = getlogrange(conf)
+    lines = getpipeoutput(['git shortlog -s %s' % revision_range]).split('\n')
+    count = 0
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        count += 1
+        if row_processor:
+            tokens = line.split()
+            commit_count = int(tokens[0])
+            author = ' '.join(tokens[1:])
+            row_processor(AuthorTotals(author, commit_count))
+    return count
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'author', 'commits'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+                gen_author_totals_data(
+                    conf,
+                    lambda row: writer.writerow([repo_name, row.author, row.total_commits]))

gitstats/data_generators/gen_file_data.py

@@ -0,0 +1,65 @@
+import csv
+import os
+import re
+
+from multiprocessing import Pool
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getcommitrange, getpipeoutput, getnumoflinesinblob
+from gitstats.data import File
+
+
+def gen_file_data(conf, row_processor):
+    '''
+    Given a configuration, pull authorship information. For
+    each author, callback to the row_processor passing an AuthorRow
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: None
+    '''
+
+    # extensions and size of files
+
+    # DBG: git ls-tree -r -l -z HEAD
+    lines = getpipeoutput(['git ls-tree -r -l -z %s' % getcommitrange(conf, 'HEAD', end_only=True)]).split(
+        '\000')
+    blobs_to_read = {} # blob_id -> File
+    for line in lines:
+        if len(line) == 0:
+            continue
+        parts = re.split('\s+', line, 4)
+        if parts[0] == '160000' and parts[3] == '-':
+            # skip submodules
+            continue
+        blob_id = parts[2]
+        size = int(parts[3])
+        fullpath = parts[4]
+        _, ext = os.path.splitext(fullpath)
+        blobs_to_read[blob_id] = File(fullpath, ext, size)
+
+    # DBG: git cat-file blob e4f17a621893811250be96c8ef9c37b5e97a1df7', 'wc -l'
+    pool = Pool(processes=conf['processes'])
+    blob_linecount = pool.map(getnumoflinesinblob, blobs_to_read.keys())
+    pool.terminate()
+    pool.join()
+    # Update cache and write down info about number of number of lines
+    for (blob_id, linecount) in blob_linecount:
+        file_data = blobs_to_read[blob_id]
+        file_data.lines = linecount
+        row_processor(file_data)
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'file', 'ext', 'size', 'line_count'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+
+                gen_file_data(
+                    conf,
+                    lambda row: writer.writerow([repo_name, row.full_path, row.ext, row.size, row.lines]))

gitstats/data_generators/gen_loc_data.py

@@ -0,0 +1,80 @@
+import csv
+import logging
+import os
+import re
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getlogrange, getpipeoutput, getstatsummarycounts
+from gitstats.data import LocByDate
+
+
+def gen_loc_data(conf, row_processor):
+    '''
+    Given a configuration, pull authorship information. For
+    each author, callback to the row_processor passing an AuthorRow
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: total lines in repo
+    '''
+
+    # line statistics
+    # outputs:
+    #  N files changed, N insertions (+), N deletions(-)
+
+    # computation of lines of code by date is better done
+    # on a linear history.
+    extra = ''
+    if conf['linear_linestats']:
+        extra = '--first-parent -m'
+
+    # DBG: git log --shortstat --first-parent -m --pretty=format:"%at %aN" --since="2017-10-01" "HEAD"'
+    lines = getpipeoutput(
+        ['git log --shortstat %s --pretty=format:"%%H %%at %%aN" %s' % (extra, getlogrange(conf, 'HEAD'))]).split('\n')
+    lines.reverse()
+    files = 0
+    inserted = 0
+    deleted = 0
+    total_lines = 0
+    for line in lines:
+        if len(line) == 0:
+            continue
+
+        if re.search('files? changed', line) is None:
+            line = line.strip()
+            if line:
+                try:
+                    parts = line.split(' ', 2)
+                    (hash, stamp, author) = (parts[0], int(parts[1]), parts[2])
+                    row_processor(LocByDate(hash, stamp, files, inserted, deleted, total_lines))
+                    files, inserted, deleted = 0, 0, 0
+                except ValueError:
+                    logging.warning(f'unexpected line "{line}')
+            else:
+                logging.warning(f'unexpected line "{line}')
+        else:
+            numbers = getstatsummarycounts(line)
+
+            if len(numbers) == 3:
+                (files, inserted, deleted) = map(lambda el: int(el), numbers)
+                total_lines += inserted
+                total_lines -= deleted
+            else:
+                logging.warning(f'Failed to handle line "{line}"')
+                (files, inserted, deleted) = (0, 0, 0)
+    return total_lines
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'sha', 'stamp', 'file count', 'lines inserted', 'lines deleted', 'total lines'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+
+                gen_loc_data(
+                    conf,
+                    lambda row: writer.writerow([repo_name, row.hash, row.stamp, row.file_count, row.lines_inserted, row.lines_deleted, row.total_lines]))

gitstats/data_generators/gen_revision_data.py

@@ -0,0 +1,75 @@
+import csv
+import os
+
+from multiprocessing import Pool
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getlogrange, getpipeoutput, getnumoffilesfromrev
+from gitstats.data import Revision
+
+
+def gen_revision_data(conf, row_processor):
+    '''
+    Given a configuration, pull revision information. For
+    each author, callback to the row_processor passing an Revision
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: Number of commits
+    '''
+
+    revisions = {} # tree_hash -> Revision
+    # Collect revision statistics
+    # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
+
+    # DBG: git rev-list --pretty=format:"%at %ai %aN <%aE>" --since="2017-10-01" "HEAD"', 'grep -v ^commit'
+    lines = getpipeoutput(
+        ['git rev-list --pretty=format:"%%T %%H %%at %%ai %%aN <%%aE>" %s' % getlogrange(conf, 'HEAD'),
+         'grep -v ^commit']).split('\n')
+    for line in lines:
+        parts = line.split(' ', 6)
+        tree_hash = parts[0]
+        sha = parts[1]
+        try:
+            stamp = int(parts[2])
+        except ValueError:
+            stamp = 0
+        timezone = parts[5]
+        author, mail = parts[6].split('<', 1)
+        author = author.strip()
+        mail = mail.rstrip('>')
+        domain = '?'
+        if mail.find('@') != -1:
+            domain = mail.rsplit('@', 1)[1]
+            domain.rstrip('>')
+        revisions[tree_hash] = Revision(sha, stamp, timezone, author, mail, domain)
+
+    # todo: consider putting in a cache for this. There was one in the original code
+    # DBG: git ls-tree -r --name-only "ceb3165b51ae0680724fd71e16a5ff836a0de41e"', 'wc -l'
+    pool = Pool(processes=conf['processes'])
+    rev_count = pool.map(getnumoffilesfromrev, revisions.keys())
+    pool.terminate()
+    pool.join()
+    # Update cache with new revisions and append then to general list
+    for (rev, count) in rev_count:
+        revision = revisions[rev]
+        revision.file_count = count
+        row_processor(revision)
+
+    return len(lines)
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'sha', 'stamp', 'timezone', 'author', 'email', 'domain', 'files_changed'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+
+                gen_revision_data(
+                    conf,
+                    lambda row: writer.writerow([repo_name, row.sha, row.stamp, row.timezone, row.author, row.email,
+                                                 row.domain, row.file_count]))

gitstats/data_generators/gen_tag_data.py

@@ -0,0 +1,74 @@
+import csv
+import os
+
+from collections import defaultdict
+from typing import Dict
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getpipeoutput
+from gitstats.data import AuthorTotals, Tag
+from gitstats.data_generators import gen_author_totals_data
+
+
+def gen_tag_data(conf, row_processor):
+    '''
+    Given a configuration, pull tag information. For
+    each tag, callback to the row_processor passing a Tag
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: None
+    '''
+
+    # tags
+    tags = {} # stamp -> tags
+    lines = getpipeoutput(['git show-ref --tags']).split('\n')
+    for line in lines:
+        if len(line) == 0:
+            continue
+        (line_hash, tag) = line.split(' ')
+
+        tag = tag.replace('refs/tags/', '')
+        output = getpipeoutput(['git log "%s" --pretty=format:"%%at" -n 1' % line_hash])
+
+        stamp = 0
+        if len(output) > 0:
+            try:
+                stamp = int(output.strip())
+            except ValueError:
+                stamp = 0
+
+        tags[stamp] = Tag(tag, stamp, line_hash, 0, {})
+
+    stamps = sorted(tags.keys())
+    prev = None
+    for stamp in stamps:
+        def process_row(row: AuthorTotals):
+            tags[stamp].authors[row.author] = row.total_commits
+            tags[stamp].commits += row.total_commits
+
+        revision_tags = tags[stamp].tag
+        if prev != None:
+            revision_tags += ' "^%s"' % prev
+
+        gen_author_totals_data(conf, process_row, revision_tags)
+        row_processor(tags[stamp])
+
+        prev = tags[stamp].tag
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'sha', 'stamp', 'commits for tag', 'author', 'commits by author'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+
+                def process_row(row):
+                    for author, commits in row.authors.items():
+                        writer.writerow([repo_name, row.hash, row.stamp, row.commits, author, commits])
+
+                gen_tag_data(conf, process_row)

gitstats/datacollector.py

@@ -5,6 +5,7 @@ import pickle
 import time
 import zlib
 
+from collections import defaultdict
 from typing import Dict
 from gitstats.data.author import Author
 
@@ -26,7 +27,7 @@ class DataCollector:
         self.activity_by_year_week = {}  # yy_wNN -> commits
         self.activity_by_year_week_peak = 0
 
-        self.authors: Dict[Author] = {}  # name -> Author
+        self.authors: Dict[Author] = defaultdict(lambda: Author())
 
         self.total_commits = 0
         self.total_files = 0

gitstats/gitdatacollector.py

@@ -1,14 +1,12 @@
 import datetime
-import logging
-import re
-import os
 
-from multiprocessing import Pool
+from collections import defaultdict
 
-from gitstats.data.author import Author
 from gitstats.datacollector import DataCollector
-from gitstats.miscfuncs import getcommitrange, getlogrange, getnumoffilesfromrev, getnumoflinesinblob, \
-    getpipeoutput, getstatsummarycounts
+from gitstats.data import Author, AuthorRow, File, LocByDate, Revision, Tag
+from gitstats.data_generators import gen_author_data, gen_author_totals_data, gen_tag_data, gen_revision_data, \
+    gen_file_data, gen_loc_data
+from gitstats.miscfuncs import getpipeoutput
 
 
 class GitDataCollector(DataCollector):
@@ -18,201 +16,77 @@ class GitDataCollector(DataCollector):
     def collect(self, directory):
         super(GitDataCollector, self).collect(directory)
 
-        self.total_authors += int(getpipeoutput(['git shortlog -s %s' % getlogrange(self.conf), 'wc -l']))
-        # self.total_lines = int(getoutput('git-ls-files -z |xargs -0 cat |wc -l'))
+        self.total_authors += self.get_total_authors()
         self.get_tags()
         self.get_revision_info()
         self.get_file_info()
         self.get_loc_info()
         self.get_author_info()
 
-    def xlate(self, name):
-        if name in self.conf['name_xlate']:
-            return self.conf['name_xlate'][name]
-        return name
+    def get_total_authors(self):
+        return gen_author_totals_data(self.conf)
 
     def get_author_info(self):
         # Per-author statistics
         # defined for stamp, author only if author commited at this timestamp.
-        self.changes_by_date_by_author = {}  # stamp -> author -> lines_added
-        # Similar to the above, but never use --first-parent
-        # (we need to walk through every commit to know who
-        # committed what, not just through mainline)
-        lines = getpipeoutput(
-            ['git log --shortstat --date-order --pretty=format:"%%at %%aN" %s' % (
-                getlogrange(self.conf, 'HEAD'))]).split('\n')
-        lines.reverse()
-        inserted = 0
-        deleted = 0
-        stamp = 0
-        for line in lines:
-            if len(line) == 0:
-                continue
-
-            # <stamp> <author>
-            if re.search('files? changed', line) is None:
-                pos = line.find(' ')
-                if pos != -1:
-                    try:
-                        oldstamp = stamp
-                        (stamp, author) = (int(line[:pos]), line[pos + 1:])
-                        author = self.xlate(author)
-                        if oldstamp > stamp:
-                            # clock skew, keep old timestamp to avoid having ugly graph
-                            stamp = oldstamp
-                        if author not in self.authors:
-                            self.authors[author] = Author()
-                        self.authors[author].commits += 1
-                        self.authors[author].lines_added += inserted
-                        self.authors[author].lines_removed += deleted
-                        if stamp not in self.changes_by_date_by_author:
-                            self.changes_by_date_by_author[stamp] = {}
-                        if author not in self.changes_by_date_by_author[stamp]:
-                            self.changes_by_date_by_author[stamp][author] = Author()
-                        self.changes_by_date_by_author[stamp][author].lines_added = self.authors[author].lines_added
-                        self.changes_by_date_by_author[stamp][author].commits = self.authors[author].commits
-                        files, inserted, deleted = 0, 0, 0
-                    except ValueError:
-                        logging.warning(f'unexpected line "{line}')
-                else:
-                    logging.warning(f'unexpected line "{line}')
-            else:
-                numbers = getstatsummarycounts(line)
 
-                if len(numbers) == 3:
-                    (files, inserted, deleted) = map(lambda el: int(el), numbers)
-                else:
-                    logging.warning(f'Failed to handle line "{line}"')
-                    (files, inserted, deleted) = (0, 0, 0)
+        self.changes_by_date_by_author = defaultdict(lambda: defaultdict(lambda: Author())) # stamp -> author -> lines_added
+
+        def row_processor(row: AuthorRow):
+            self.authors[row.author].commits += 1
+            self.authors[row.author].lines_added += row.lines_inserted
+            self.authors[row.author].lines_removed += row.lines_deleted
+            self.changes_by_date_by_author[row.stamp][row.author].lines_added = self.authors[row.author].lines_added
+            self.changes_by_date_by_author[row.stamp][row.author].commits = self.authors[row.author].commits
+
+        gen_author_data(self.conf, row_processor)
 
     def get_loc_info(self):
-        # line statistics
-        # outputs:
-        #  N files changed, N insertions (+), N deletions(-)
-        # <stamp> <author>
+
         self.changes_by_date = {}  # stamp -> { files, ins, del }
-        # computation of lines of code by date is better done
-        # on a linear history.
-        extra = ''
-        if self.conf['linear_linestats']:
-            extra = '--first-parent -m'
-        lines = getpipeoutput(
-            ['git log --shortstat %s --pretty=format:"%%at %%aN" %s' % (extra, getlogrange(self.conf, 'HEAD'))]).split(
-            '\n')
-        lines.reverse()
-        files = 0
-        inserted = 0
-        deleted = 0
-        total_lines = 0
-        for line in lines:
-            if len(line) == 0:
-                continue
-
-            # <stamp> <author>
-            if re.search('files? changed', line) is None:
-                pos = line.find(' ')
-                if pos != -1:
-                    try:
-                        (stamp, author) = (int(line[:pos]), line[pos + 1:])
-                        self.changes_by_date[stamp] = {'files': files, 'ins': inserted, 'del': deleted,
-                                                       'lines': total_lines}
-
-                        date = datetime.datetime.fromtimestamp(stamp)
-                        yymm = date.strftime('%Y-%m')
-                        self.lines_added_by_month[yymm] = self.lines_added_by_month.get(yymm, 0) + inserted
-                        self.lines_removed_by_month[yymm] = self.lines_removed_by_month.get(yymm, 0) + deleted
-
-                        yy = date.year
-                        self.lines_added_by_year[yy] = self.lines_added_by_year.get(yy, 0) + inserted
-                        self.lines_removed_by_year[yy] = self.lines_removed_by_year.get(yy, 0) + deleted
-
-                        files, inserted, deleted = 0, 0, 0
-                    except ValueError:
-                        logging.warning(f'unexpected line "{line}')
-                else:
-                    logging.warning(f'unexpected line "{line}')
-            else:
-                numbers = getstatsummarycounts(line)
+        def row_processor(row: LocByDate):
+            self.changes_by_date[row.stamp] = {
+                'files': row.file_count,
+                'ins': row.lines_inserted,
+                'del': row.lines_deleted,
+                'lines': row.total_lines
+            }
+            date = datetime.datetime.fromtimestamp(row.stamp)
+            yymm = date.strftime('%Y-%m')
+            self.lines_added_by_month[yymm] = self.lines_added_by_month.get(yymm, 0) + row.lines_inserted
+            self.lines_removed_by_month[yymm] = self.lines_removed_by_month.get(yymm, 0) + row.lines_deleted
+
+            yy = date.year
+            self.lines_added_by_year[yy] = self.lines_added_by_year.get(yy, 0) + row.lines_inserted
+            self.lines_removed_by_year[yy] = self.lines_removed_by_year.get(yy, 0) + row.lines_deleted
 
-                if len(numbers) == 3:
-                    (files, inserted, deleted) = map(lambda el: int(el), numbers)
-                    total_lines += inserted
-                    total_lines -= deleted
-                    self.total_lines_added += inserted
-                    self.total_lines_removed += deleted
+            self.total_lines_added += row.lines_inserted
+            self.total_lines_removed += row.lines_deleted
 
-                else:
-                    logging.warning(f'Failed to handle line "{line}"')
-                    (files, inserted, deleted) = (0, 0, 0)
-            # self.changes_by_date[stamp] = { 'files': files, 'ins': inserted, 'del': deleted }
-        self.total_lines += total_lines
+        self.total_lines += gen_loc_data(self.conf, row_processor)
 
     def get_file_info(self):
         # extensions and size of files
-        lines = getpipeoutput(['git ls-tree -r -l -z %s' % getcommitrange(self.conf, 'HEAD', end_only=True)]).split(
-            '\000')
-        blobs_to_read = []
-        for line in lines:
-            if len(line) == 0:
-                continue
-            parts = re.split('\s+', line, 4)
-            if parts[0] == '160000' and parts[3] == '-':
-                # skip submodules
-                continue
-            blob_id = parts[2]
-            size = int(parts[3])
-            fullpath = parts[4]
-
-            self.total_size += size
+        def row_processor(row: File):
+            self.total_size += row.size
             self.total_files += 1
+            if row.ext not in self.extensions:
+                self.extensions[row.ext] = {'files': 0, 'lines': 0}
+            self.extensions[row.ext]['files'] += 1
+            self.extensions[row.ext]['lines'] += row.lines
 
-            _, ext = os.path.splitext(fullpath)
-            if len(ext) > self.conf['max_ext_length']:
-                ext = ''
-            if ext not in self.extensions:
-                self.extensions[ext] = {'files': 0, 'lines': 0}
-            self.extensions[ext]['files'] += 1
-            # if cache empty then add ext and blob id to list of new blob's
-            # otherwise try to read needed info from cache
-            if 'lines_in_blob' not in self.cache.keys():
-                blobs_to_read.append((ext, blob_id))
-                continue
-            if blob_id in self.cache['lines_in_blob'].keys():
-                self.extensions[ext]['lines'] += self.cache['lines_in_blob'][blob_id]
-            else:
-                blobs_to_read.append((ext, blob_id))
-        # Get info abount line count for new blob's that wasn't found in cache
-        pool = Pool(processes=self.conf['processes'])
-        ext_blob_linecount = pool.map(getnumoflinesinblob, blobs_to_read)
-        pool.terminate()
-        pool.join()
-        # Update cache and write down info about number of number of lines
-        for (ext, blob_id, linecount) in ext_blob_linecount:
-            if 'lines_in_blob' not in self.cache:
-                self.cache['lines_in_blob'] = {}
-            self.cache['lines_in_blob'][blob_id] = linecount
-            self.extensions[ext]['lines'] += self.cache['lines_in_blob'][blob_id]
+        gen_file_data(self.conf, row_processor)
 
     def get_revision_info(self):
         # Collect revision statistics
         # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
-        lines = getpipeoutput(
-            ['git rev-list --pretty=format:"%%at %%ai %%aN <%%aE>" %s' % getlogrange(self.conf, 'HEAD'),
-             'grep -v ^commit']).split(
-            '\n')
-        for line in lines:
-            parts = line.split(' ', 4)
-            try:
-                stamp = int(parts[0])
-            except ValueError:
-                stamp = 0
-            timezone = parts[3]
-            author, mail = parts[4].split('<', 1)
-            author = self.xlate(author.rstrip())
-            mail = mail.rstrip('>')
-            domain = '?'
-            if mail.find('@') != -1:
-                domain = mail.rsplit('@', 1)[1]
+
+        def row_processor(row: Revision):
+            stamp = row.stamp
+            domain = row.domain
+            author = row.author
+            timezone = row.timezone
+
             date = datetime.datetime.fromtimestamp(float(stamp))
 
             # First and last commit stamp (may be in any order because of cherry-picking and patches)
@@ -258,8 +132,6 @@ class GitDataCollector(DataCollector):
                 self.activity_by_year_week_peak = self.activity_by_year_week[yyw]
 
             # author stats
-            if author not in self.authors:
-                self.authors[author] = Author()
             self.authors[author].activity_by_day_and_hour[day][hour] += 1
             # commits, note again that commits may be in any date order because of cherry-picking and patches
             if not self.authors[author].last_commit_stamp:
@@ -303,99 +175,36 @@ class GitDataCollector(DataCollector):
 
             # timezone
             self.commits_by_timezone[timezone] = self.commits_by_timezone.get(timezone, 0) + 1
-        # outputs "<stamp> <files>" for each revision
-        revlines = getpipeoutput(
-            ['git rev-list --pretty=format:"%%at %%T" %s' % getlogrange(self.conf, 'HEAD'),
-             'grep -v ^commit']).strip().split('\n')
-        lines = []
-        revs_to_read = []
-        # Look up rev in cache and take info from cache if found
-        # If not append rev to list of rev to read from repo
-        for revline in revlines:
-            time, rev = revline.split(' ')
-            # if cache empty then add time and rev to list of new rev's
-            # otherwise try to read needed info from cache
-            if 'files_in_tree' not in self.cache.keys():
-                revs_to_read.append((time, rev))
-                continue
-            if rev in self.cache['files_in_tree'].keys():
-                lines.append('%d %d' % (int(time), self.cache['files_in_tree'][rev]))
-            else:
-                revs_to_read.append((time, rev))
-        # Read revisions from repo
-        pool = Pool(processes=self.conf['processes'])
-        time_rev_count = pool.map(getnumoffilesfromrev, revs_to_read)
-        pool.terminate()
-        pool.join()
-        # Update cache with new revisions and append then to general list
-        for (time, rev, count) in time_rev_count:
-            if 'files_in_tree' not in self.cache:
-                self.cache['files_in_tree'] = {}
-            self.cache['files_in_tree'][rev] = count
-            lines.append('%d %d' % (int(time), count))
-        self.total_commits += len(lines)
-        for line in lines:
-            parts = line.split(' ')
-            if len(parts) != 2:
-                continue
-            (stamp, files) = parts[0:2]
-            try:
-                self.files_by_stamp[int(stamp)] = int(files)
-            except ValueError:
-                logging.warning(f'Failed to parse line "{line}"')
+
+            # file counts
+            self.files_by_stamp[stamp] = row.file_count
+
+        self.total_commits += gen_revision_data(self.conf, row_processor)
 
     def get_tags(self):
-        # tags
-        lines = getpipeoutput(['git show-ref --tags']).split('\n')
-        for line in lines:
-            if len(line) == 0:
-                continue
-            (line_hash, tag) = line.split(' ')
-
-            tag = tag.replace('refs/tags/', '')
-            output = getpipeoutput(['git log "%s" --pretty=format:"%%at %%aN" -n 1' % line_hash])
-            if len(output) > 0:
-                parts = output.split(' ')
-                try:
-                    stamp = int(parts[0])
-                except ValueError:
-                    stamp = 0
-                self.tags[tag] = {'stamp': stamp,
-                                  'hash': line_hash,
-                                  'date': datetime.datetime.fromtimestamp(stamp).strftime('%Y-%m-%d'),
-                                  'commits': 0,
-                                  'authors': {}}
-        # collect info on tags, starting from latest
-        tags_sorted_by_date_asc = [tup[1] for tup in sorted([(el[1]['date'], el[0]) for el in self.tags.items()])]
-        # tags_sorted_by_date_desc = map(lambda el: el[1],
-        #                                reversed(sorted(map(lambda el: (el[1]['date'], el[0]), self.tags.items()))))
-        prev = None
-        #        for tag in reversed(tags_sorted_by_date_desc):
-        for tag in tags_sorted_by_date_asc:
-            cmd = 'git shortlog -s "%s"' % tag
-            if prev is not None:
-                cmd += ' "^%s"' % prev
-            output = getpipeoutput([cmd])
-            if len(output) == 0:
-                continue
-            prev = tag
-            for line in output.split('\n'):
-                parts = re.split('\s+', line, 2)
-                commits = int(parts[1])
-                author = parts[2]
-                self.tags[tag]['commits'] += commits
-                self.tags[tag]['authors'][author] = commits
+        def row_processor(row: Tag):
+            self.tags[row.tag] = {
+                'stamp': row.stamp,
+                'hash': row.hash,
+                'date': datetime.datetime.fromtimestamp(row.stamp).strftime('%Y-%m-%d'),
+                'commits': row.commits,
+                'authors': row.authors
+            }
+
+        gen_tag_data(self.conf, row_processor)
 
     def refine(self):
         # authors
         # name -> {place_by_commits, commits_frac, date_first, date_last, timedelta}
         self.authors_by_commits = self.getAuthors()
+        total_commits_without_merge = 0
         for i, name in enumerate(self.authors_by_commits):
             self.authors[name].place_by_commits = i + 1
+            total_commits_without_merge += self.authors[name].commits
 
         for name in self.authors.keys():
             a = self.authors[name]
-            a.commits_frac = (100 * float(a.commits)) / self.getTotalCommits()
+            a.commits_frac = (100 * float(a.commits)) / total_commits_without_merge
             date_first = datetime.datetime.fromtimestamp(a.first_commit_stamp)
             date_last = datetime.datetime.fromtimestamp(a.last_commit_stamp)
             delta = date_last - date_first

gitstats/gitstats.py

@@ -1,138 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2007-2014 Heikki Hokkanen <hoxu@users.sf.net> & others (see doc/AUTHOR)
-# GPLv2 / GPLv3
-import getopt
-import logging
-import os
-import sys
-import time
-
-import multiprocessing_logging
-
-from collections import defaultdict
-
-from .gitdatacollector import GitDataCollector
-from .htmlreportcreator import HTMLReportCreator
-from .miscfuncs import getgnuplotversion
-
-exectime_internal = 0.0
-exectime_external = 0.0
-
-conf = {
-    'max_domains': 10,
-    'max_ext_length': 10,
-    'style': 'gitstats.css',
-    'max_authors': 20,
-    'authors_top': 5,
-    'commit_begin': '',
-    'commit_end': 'HEAD',
-    'linear_linestats': 1,
-    'project_name': '',
-    'processes': 8,
-    'start_date': '',
-    'logging': logging.INFO,
-    'name_xlate': defaultdict(dict)
-}
-
-class GitStats:
-    def _usage(self):
-        print(f"""
-    Usage: gitstats [options] <gitpath..> <outputpath>
-
-    Options:
-    -c key=value     Override configuration value
-    -n key=value     Define author name equivalency (key will treated the same as value)
-
-    Default config values:
-    {conf}
-
-    Please see the manual page for more details.
-    """)
-
-    def run(self):
-        optlist, args = getopt.getopt(sys.argv[1:], 'hc:n:', ["help"])
-        for o, v in optlist:
-            if o == '-c':
-                key, value = v.split('=', 1)
-                if key not in conf:
-                    raise KeyError('no such key "%s" in config' % key)
-                if isinstance(conf[key], int):
-                    conf[key] = int(value)
-                else:
-                    conf[key] = value
-            elif o in ('-h', '--help'):
-                self._usage()
-                sys.exit()
-            elif o == '-n':
-                key, value = v.split('=', 1)
-                conf['name_xlate'][key] = value
-
-
-        if len(args) < 2:
-            self._usage()
-            sys.exit(0)
-
-        outputpath = os.path.abspath(args[-1])
-        paths = args[0:-1]
-        outputpath = os.path.abspath(outputpath)
-
-        logging.basicConfig(level=conf['logging'], format='%(message)s')
-        multiprocessing_logging.install_mp_handler()
-        time_start = time.time()
-
-
-        rundir = os.getcwd()
-
-        try:
-            os.makedirs(outputpath)
-        except OSError:
-            pass
-        if not os.path.isdir(outputpath):
-            logging.fatal('Output path is not a directory or does not exist')
-            sys.exit(1)
-
-        if not getgnuplotversion():
-            logging.error('gnuplot not found')
-            sys.exit(1)
-
-        logging.info(f'Output path: {outputpath}')
-        cachefile = os.path.join(outputpath, 'gitstats.cache')
-
-        data = GitDataCollector(conf)
-        data.loadCache(cachefile)
-
-        for gitpath in paths:
-            logging.info(f'Git path: {gitpath}')
-
-            prevdir = os.getcwd()
-            os.chdir(gitpath)
-
-            logging.info('Collecting data...')
-            data.collect(gitpath)
-
-            os.chdir(prevdir)
-
-        data.saveCache(cachefile)
-
-        logging.info('Refining data...')
-        data.refine()
-
-        os.chdir(rundir)
-
-        logging.info('Generating report...')
-        report = HTMLReportCreator(conf)
-        report.create(data, outputpath)
-
-        time_end = time.time()
-        calculated_exectime_internal = time_end - time_start
-        logging.info(f'Execution time {calculated_exectime_internal} secs, {exectime_external} secs ({(100.0 * exectime_external) / calculated_exectime_internal}%) in external commands)')
-
-        print('You may now run:')
-        print()
-        print('   sensible-browser \'%s\'' % os.path.join(outputpath, 'index.html').replace("'", "'\\''"))
-        print()
-
-
-if __name__ == '__main__':
-    g = GitStats()
-    g.run()

gitstats/miscfuncs.py

@@ -80,19 +80,20 @@ def getgnuplotversion():
     return getpipeoutput(['%s --version' % gnuplot_cmd]).split('\n')[0]
 
 
-def getnumoffilesfromrev(time_rev):
+def getnumoffilesfromrev(tree_hash):
     """
     Get number of files changed in commit
     """
-    time_portion, rev = time_rev
-    return (int(time_portion), rev, int(getpipeoutput(['git ls-tree -r --name-only "%s"' % rev, 'wc -l']).split('\n')[0]))
+    # DBG: git ls-tree -r --name-only "ceb3165b51ae0680724fd71e16a5ff836a0de41e"' | 'wc -l'
+    return (tree_hash, int(getpipeoutput(['git ls-tree -r --name-only "%s"' % tree_hash, 'wc -l']).split('\n')[0]))
 
 
-def getnumoflinesinblob(ext_blob):
+def getnumoflinesinblob(blob_id):
     """
     Get number of lines in blob
     """
-    ext, blob_id = ext_blob
-    return ext, blob_id, int(getpipeoutput(['git cat-file blob %s' % blob_id, 'wc -l']).split()[0])
+
+    # DBG: git cat-file blob e4f17a621893811250be96c8ef9c37b5e97a1df7', 'wc -l'
+    return blob_id, int(getpipeoutput(['git cat-file blob %s' % blob_id, 'wc -l']).split()[0])
 
 

gitstats/process.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+# Copyright (c) 2007-2014 Heikki Hokkanen <hoxu@users.sf.net> & others (see doc/AUTHOR)
+# GPLv2 / GPLv3
+import logging
+import os
+import sys
+import time
+
+import multiprocessing_logging
+
+from gitstats.gitdatacollector import GitDataCollector
+from gitstats.htmlreportcreator import HTMLReportCreator
+from gitstats.miscfuncs import getgnuplotversion
+from gitstats import cli
+
+exectime_internal = 0.0
+exectime_external = 0.0
+
+def run():
+
+    conf, paths, outputpath = cli.get_cli()
+
+    logging.basicConfig(level=conf['logging'], format='%(message)s')
+    multiprocessing_logging.install_mp_handler()
+    time_start = time.time()
+
+
+    rundir = os.getcwd()
+
+    try:
+        os.makedirs(outputpath)
+    except OSError:
+        pass
+    if not os.path.isdir(outputpath):
+        logging.fatal('Output path is not a directory or does not exist')
+        sys.exit(1)
+
+    if not getgnuplotversion():
+        logging.error('gnuplot not found')
+        sys.exit(1)
+
+    logging.info(f'Output path: {outputpath}')
+    cachefile = os.path.join(outputpath, 'gitstats.cache')
+
+    data = GitDataCollector(conf)
+    data.loadCache(cachefile)
+
+    for gitpath in paths:
+        logging.info(f'Git path: {gitpath}')
+
+        prevdir = os.getcwd()
+        os.chdir(gitpath)
+
+        logging.info('Collecting data...')
+        data.collect(gitpath)
+
+        os.chdir(prevdir)
+
+    data.saveCache(cachefile)
+
+    logging.info('Refining data...')
+    data.refine()
+
+    os.chdir(rundir)
+
+    logging.info('Generating report...')
+    report = HTMLReportCreator(conf)
+    report.create(data, outputpath)
+
+    time_end = time.time()
+    calculated_exectime_internal = time_end - time_start
+    logging.info(f'Execution time {calculated_exectime_internal} secs, {exectime_external} secs ({(100.0 * exectime_external) / calculated_exectime_internal}%) in external commands)')
+
+    print('You may now run:')
+    print()
+    print('   sensible-browser \'%s\'' % os.path.join(outputpath, 'index.html').replace("'", "'\\''"))
+    print()
+
+
+if __name__ == '__main__':
+    run()