Significant rework:

* Read in full revision graph
* Use Tokei to get detailed LOC measurements for each rev
* Trim down number of csv files created

.gitignore

@@ -1,2 +1,4 @@
 **/__pycache__
 *.egg-info
+build
+dist

gitstats/cli.py

@@ -3,6 +3,8 @@ import logging
 import os
 import sys
 
+from datetime import datetime, timezone
+
 conf = {
     'max_domains': 10,
     'max_ext_length': 10,
@@ -15,6 +17,7 @@ conf = {
     'project_name': '',
     'processes': 8,
     'start_date': '',
+    'end_date': '',
     'logging': logging.INFO,
     'resrouce_file_pattern': '**/resources/**/*',
 }
@@ -58,3 +61,14 @@ def get_cli():
     outputpath = os.path.abspath(outputpath)
 
     return conf, paths, outputpath
+
+def get_begin_end_timestamps(conf):
+    if 'start_date' in conf and conf['start_date']:
+        begin = int(datetime.strptime(conf['start_date'], '%Y-%m-%d').replace(tzinfo=timezone.utc).timestamp())
+    else:
+        begin = 0
+    if 'end_date' in conf and conf['end_date']:
+        end = int(datetime.strptime(conf['end_date'], '%Y-%m-%d').replace(tzinfo=timezone.utc).timestamp())
+    else:
+        end = 99999999999
+    return begin, end

gitstats/data/__init__.py

@@ -5,4 +5,6 @@ from .tag import Tag
 from .revision import Revision
 from .file import File
 from .loc_by_date import LocByDate
-from .pr import PullRequest
+from .pr import PullRequest
+from .file_info import FileInfo
+from .revision_graph import RevisionGraph

gitstats/data/file_info.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+@dataclass
+class FileInfo:
+    language: str
+    file_count: int
+    line_count: int
+    code_line_count: int
+    comment_line_count: int
+    blank_line_count: int
+
+    def __post_init__(self):
+        self.file_count = int(self.file_count)
+        self.line_count = int(self.line_count)
+        self.code_line_count = int(self.code_line_count)
+        self.comment_line_count = int(self.comment_line_count)
+        self.blank_line_count = int(self.blank_line_count)
+
+    def __sub__(self, other: 'FileInfo') -> 'FileInfo':
+        return FileInfo(self.language,
+                        self.file_count - other.file_count,
+                        self.line_count - other.line_count,
+                        self.code_line_count - other.code_line_count,
+                        self.comment_line_count - other.comment_line_count,
+                        self.blank_line_count - other.blank_line_count)

gitstats/data/revision.py

@@ -1,9 +1,7 @@
-from collections import defaultdict
 from dataclasses import dataclass, field
+from .file_info import FileInfo
 from typing import Dict
 
-#    # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
-
 @dataclass
 class Revision:
     hash: str
@@ -12,5 +10,10 @@ class Revision:
     author: str = ''
     email: str = ''
     domain: str = ''
-    file_count: int = 0
-
+    comments: str = ''
+    master_pr: int = 0
+    branch_parent: str = ''
+    master_parent: str = ''
+    file_infos: Dict[str, FileInfo] = field(default_factory=lambda: {})
+    delta: Dict[str, FileInfo] = field(default_factory=lambda: {})
+    valid_pr: bool = True

gitstats/data/revision_graph.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+from typing import Dict, List, Set
+from gitstats.data.revision import Revision
+
+@dataclass
+class RevisionGraph:
+    revisions: Dict[str, Revision]
+    master_revs: Set[str]
+    linkage: Dict[str, List[str]]
+
+    def add_revision_to_graph(self, revision: Revision, parents: List[str], is_master: bool=False):
+        if not revision.hash in self.revisions:
+            self.revisions[revision.hash] = revision
+        if not revision.hash in self.linkage:
+            self.linkage[revision.hash] = parents
+        if revision.master_pr or is_master:
+            self.master_revs.add(revision.hash)

gitstats/data_generators/__init__.py

@@ -4,4 +4,6 @@ from .gen_tag_data import gen_tag_data
 from .gen_revision_data import gen_revision_data
 from .gen_file_data import gen_file_data
 from .gen_loc_data import gen_loc_data
-from .gen_pr_data import gen_pr_data
+from .gen_pr_data import gen_pr_data
+from .gen_revision_graph import gen_revision_graph
+from .gen_complete_file_info import gen_complete_file_info

gitstats/data_generators/gen_complete_file_info.py

@@ -0,0 +1,76 @@
+import csv
+import os
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getpipeoutput
+from gitstats.data import FileInfo, Revision, RevisionGraph
+from gitstats.data_generators import gen_revision_graph
+
+
+def gen_complete_file_info(graph: RevisionGraph):
+    '''
+    Given a dictionary of revisions on the master branch, collect all file info
+    using tokei for that revision
+
+    :param: master_rev - a dictionary of commit hash to Revision object for revisions on the master branch
+
+    :return: None. As a side effect, compliete file info by language type will be added to all
+    revisions in master_rev
+    '''
+
+    # use tokei to gather detailed file info for each revision on master
+    for revision in graph.master_revs:
+        getpipeoutput([f'git checkout {revision}'])
+        # for some reason if we combine these, tokei gives incorrect results!!!!
+        lines = getpipeoutput(['tokei']).split('\n')
+        for line in lines[3:-3] + [lines[-2]]:
+            line = line.strip()
+            file_info = FileInfo(*line.rsplit(maxsplit=5))
+            graph.revisions[revision].file_infos[file_info.language] = file_info
+
+    getpipeoutput(['git checkout master'])
+
+    # run through master revisions and calculate delta with previous master revision
+    for revision in graph.master_revs:
+        master_parent = graph.revisions[revision].master_parent
+        if master_parent in graph.master_revs:
+            current = graph.revisions[revision].file_infos
+            previous = graph.revisions[master_parent].file_infos
+            for lang, cur_file_info in current.items():
+                if lang in previous:
+                    graph.revisions[revision].delta[lang] = cur_file_info - previous[lang]
+                else:
+                    graph.revisions[revision].delta[lang] = cur_file_info
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'hash', 'stamp', 'author', 'language', 'files', 'lines', 'code', 'comments', 'blanks'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+                graph = gen_revision_graph()
+                gen_complete_file_info(graph)
+
+                for rev in graph.master_revs:
+                    revision: Revision = graph.revisions[rev]
+                    for lang, file_info in revision.delta.items():
+                        if file_info.file_count or \
+                                file_info.line_count or \
+                                file_info.code_line_count or \
+                                file_info.comment_line_count or \
+                                file_info.blank_line_count:
+                            writer.writerow([repo_name,
+                                             revision.hash,
+                                             revision.stamp,
+                                             graph.revisions[revision.branch_parent].author,
+                                             lang,
+                                             file_info.file_count,
+                                             file_info.line_count,
+                                             file_info.code_line_count,
+                                             file_info.comment_line_count,
+                                             file_info.blank_line_count])

gitstats/data_generators/gen_pr_data.py

@@ -2,82 +2,41 @@ import csv
 import logging
 import os
 
-from multiprocessing import Pool
+from datetime import datetime
 
 from gitstats import cli, cd
-from gitstats.miscfuncs import getlogrange, getpipeoutput, gettimedelta
-from gitstats.data import PullRequest
+from gitstats.data import Revision, PullRequest, RevisionGraph
+from gitstats.data_generators import gen_revision_graph
 
 
-def gen_pr_data(conf, row_processor):
+def gen_pr_data(row_processor, graph: RevisionGraph):
     '''
     Given a configuration, pull revision information. For
     each author, callback to the row_processor passing an PullRequest
 
-    :param conf: configuration (mostly used for date limits)
+    As a side effect, every revision in the master_rev dictionary will be updated
+    with it's branch_parent and master_parent
+
     :param row_processor: function to receive the callback
     :return: None
     '''
 
-    prs = {} # hash -> PullRequest
-
-    # DBG: git log --all --grep="Merge pull request .* to master" --shortstat --pretty=format:"%H %at %aN" --since="2017-10-01" "HEAD"', 'grep -v ^commit'
-    lines = getpipeoutput(
-        ['git log --all --grep="Merge pull request .* to master" --shortstat '
-         '--pretty=format:"%%H %%at %%aN|%%P" %s' % getlogrange(conf, 'HEAD'),
-         'grep -v ^"files changed"']).split('\n')
-    for line in lines:
-        line = line.strip()
-        if line and not 'files changed' in line:
-            parts = line.split(' ', 2)
-            hash = parts[0]
-            try:
-                stamp = int(parts[1])
-            except ValueError:
-                stamp = 0
-            (author, parent_hashes) = parts[2].split('|')
-            parent_hashes = parent_hashes.split(' ')
-            if len(parent_hashes) == 2:
-                prs[hash] = PullRequest(stamp, hash, author, parent_hashes)
-
-    keys = prs.keys()
-    for pr in prs.values():
-        if pr.parent_hashes[0] in keys:
-            pr.master_rev = pr.parent_hashes[0]
-            if pr.parent_hashes[1] in keys:
-                logging.warning(f"Unexpected branching: {pr}")
-                pr.invalid_pr = True
-            else:
-                pr.branch_rev = pr.parent_hashes[1]
-        else:
-            pr.branch_rev = pr.parent_hashes[0]
-            if pr.parent_hashes[1] in keys:
-                pr.master_rev = pr.parent_hashes[1]
+    for rev in graph.master_revs:
+        revision = graph.revisions[rev]
+        if revision.valid_pr and revision.branch_parent in graph.revisions:
+            branch_rev: Revision = graph.revisions[revision.branch_parent]
+            delta = datetime.utcfromtimestamp(revision.stamp) - datetime.utcfromtimestamp(branch_rev.stamp)
+            if delta.total_seconds() < 0:
+                logging.warning(f"Unexpected. Negative duration: {rev}")
+                revision.valid_pr = False
             else:
-                logging.warning(f"Unexpected branching: {pr}")
-                pr.invalid_pr = True
-
-    prs_to_query = [(pr.hash, pr.stamp, pr.branch_rev) for pr in prs.values() if not pr.invalid_pr]
-
-    # # todo: consider putting in a cache for this. There was one in the original code
-    # # DBG:  git log -n 1 --format=%at "ceb3165b51ae0680724fd71e16a5ff836a0de41e"
-    pool = Pool(processes=conf['processes'])
-    time_deltas = pool.map(gettimedelta, prs_to_query)
-    pool.terminate()
-    pool.join()
-    for (hash, timedelta) in time_deltas:
-        pr = prs[hash]
-        pr.duration = timedelta
-        if pr.duration.total_seconds() < 0:
-            pr.invalid_pr = True
-            logging.warning(f"Unexpected. Negative duration: {pr}")
-        else:
-            row_processor(pr)
-
+                row_processor(PullRequest(revision.stamp, revision.hash, revision.author,
+                                          graph.linkage[rev], revision.branch_parent, rev, delta))
 
 
 if __name__ == "__main__":
     conf, paths, outputpath = cli.get_cli()
+    begin, end = cli.get_begin_end_timestamps(conf)
     with open(outputpath, 'w', encoding='utf8') as f:
         writer = csv.writer(f)
         writer.writerow(['repo', 'hash', 'stamp', 'masterRev', 'branchRev', 'prMergeDuration', 'prMergeDurationHr'])
@@ -85,6 +44,9 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
+                graph = gen_revision_graph()
+
                 def row_processor(row: PullRequest):
-                    writer.writerow([repo_name, row.hash, row.stamp, row.master_rev, row.branch_rev, row.duration.total_seconds(), row.duration])
-                gen_pr_data(conf, row_processor)
+                    if row.stamp >= begin and row.stamp <= end:
+                        writer.writerow([repo_name, row.hash, row.stamp, row.master_rev, row.branch_rev, row.duration.total_seconds(), row.duration])
+                gen_pr_data(row_processor, graph)

gitstats/data_generators/gen_revision_data.py

@@ -4,74 +4,46 @@ import os
 from multiprocessing import Pool
 
 from gitstats import cli, cd
-from gitstats.miscfuncs import getlogrange, getpipeoutput, getnumoffilesfromrev
-from gitstats.data import Revision
+from gitstats.miscfuncs import getnumoffilesfromrev
+from gitstats.data import Revision, RevisionGraph
+from gitstats.data_generators.gen_revision_graph import gen_revision_graph
 
 
-def gen_revision_data(conf, row_processor):
+def gen_revision_data(conf, row_processor, graph: RevisionGraph):
     '''
     Given a configuration, pull revision information. For
     each author, callback to the row_processor passing an Revision
 
     :param conf: configuration (mostly used for date limits)
     :param row_processor: function to receive the callback
-    :return: Number of commits
+    :return: None
     '''
 
-    revisions = {} # tree_hash -> Revision
-    # Collect revision statistics
-    # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'"
-
-    # DBG: git rev-list --pretty=format:"%at %ai %aN <%aE>" --since="2017-10-01" "HEAD"', 'grep -v ^commit'
-    lines = getpipeoutput(
-        ['git rev-list --pretty=format:"%%T %%H %%at %%ai %%aN <%%aE>" %s' % getlogrange(conf, 'HEAD'),
-         'grep -v ^commit']).split('\n')
-    for line in lines:
-        line = line.strip()
-        if line:
-            parts = line.split(' ', 6)
-            tree_hash = parts[0]
-            sha = parts[1]
-            try:
-                stamp = int(parts[2])
-            except ValueError:
-                stamp = 0
-            timezone = parts[5]
-            author, mail = parts[6].split('<', 1)
-            author = author.strip()
-            mail = mail.rstrip('>')
-            domain = '?'
-            if mail.find('@') != -1:
-                domain = mail.rsplit('@', 1)[1]
-                domain.rstrip('>')
-            revisions[tree_hash] = Revision(sha, stamp, timezone, author, mail, domain)
-
-    if revisions:
-        # todo: consider putting in a cache for this. There was one in the original code
-        # DBG: git ls-tree -r --name-only "ceb3165b51ae0680724fd71e16a5ff836a0de41e"', 'wc -l'
-        pool = Pool(processes=conf['processes'])
-        rev_count = pool.map(getnumoffilesfromrev, revisions.keys())
-        pool.terminate()
-        pool.join()
-        # Update cache with new revisions and append then to general list
-        for (rev, count) in rev_count:
-            revision = revisions[rev]
-            revision.file_count = count
-            row_processor(revision)
-
-    return len(lines)
+    # todo: consider putting in a cache for this. There was one in the original code
+    # DBG: git ls-tree -r --name-only "ceb3165b51ae0680724fd71e16a5ff836a0de41e"', 'wc -l'
+    pool = Pool(processes=conf['processes'])
+    rev_count = pool.map(getnumoffilesfromrev, graph.revisions.keys())
+    pool.terminate()
+    pool.join()
+    # Update cache with new revisions and append then to general list
+    for (rev, count) in rev_count:
+        revision = graph.revisions[rev]
+        revision.file_count = count
+        row_processor(revision)
 
 
 if __name__ == "__main__":
     conf, paths, outputpath = cli.get_cli()
+    begin, end = cli.get_begin_end_timestamps(conf)
     with open(outputpath, 'w', encoding='utf8') as f:
         writer = csv.writer(f)
         writer.writerow(['repo', 'sha', 'stamp', 'timezone', 'author', 'email', 'domain', 'files_changed'])
 
         for path in paths:
+            rev_by_tree_hash, _, _, _ = gen_revision_graph(begin, end)
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
                 def row_processor(row: Revision):
                     writer.writerow([repo_name, row.hash, row.stamp, row.timezone, row.author, row.email,
                                      row.domain, row.file_count])
-                gen_revision_data(conf, row_processor)
+                gen_revision_data(conf, row_processor, rev_by_tree_hash)

gitstats/data_generators/gen_revision_graph.py

@@ -0,0 +1,114 @@
+import logging
+import os
+import re
+
+from typing import Dict
+from gitstats import cli, cd
+from gitstats.miscfuncs import getpipeoutput
+from gitstats.data import Revision, RevisionGraph
+
+
+def gen_revision_graph() -> RevisionGraph:
+    '''
+    Given beginning and ending time stamp, get all revisions from the repo within that range,
+    key them by tree_hash, commit_hash as well as create a graph of revisions and a list
+    of revisions merging to master
+
+    :return: RevisionGraph
+    '''
+
+    # this match string for PRs merged to master is particular to BitBucket
+    # probably should come from configuration
+    prog = re.compile(r'Merge pull request #([0-9]*) in.*to master')
+
+    graph = RevisionGraph({}, set(), {})
+
+    lines = getpipeoutput(
+        [f'git rev-list --pretty="%T|%H|%at|%ai|%aN|%aE|%P|%s" "HEAD"',
+         'grep -v ^commit']).split('\n')
+    for line in lines:
+        line = line.strip()
+        if line:
+            graph.add_revision_to_graph(*get_revision_from_line(line, prog))
+
+    new_masters = set()
+    for rev in graph.master_revs:
+        parents = graph.linkage[rev]
+        revision: Revision = graph.revisions[rev]
+        for parent in parents:
+            if parent in graph.master_revs:
+                if revision.master_parent:
+                    logging.warning(f"{rev} has multiple master parents")
+                    revision.valid_pr = False
+                revision.master_parent = parent
+            else:
+                if revision.branch_parent:
+                    if not revision.master_parent:
+                        # we likely have a merge into master in a branch that didn't use
+                        # bitbucket conventions... arbitrarily choose the oldest parent
+                        # revision as the master branch (we could back chain both and find
+                        # which branch exists in the ancestry of the other, but for now,
+                        # this will suffice
+                        if graph.revisions[parent].stamp < graph.revisions[revision.branch_parent].stamp:
+                            revision.master_parent = parent
+                            new_masters.add(parent)
+                        else:
+                            revision.master_parent = revision.branch_parent
+                            new_masters.add(revision.branch_parent)
+                            revision.branch_parent = parent
+                    else:
+                        logging.warning(f"{rev} has multiple branch parents")
+                        revision.valid_pr = False
+                else:
+                    revision.branch_parent = parent
+    graph.master_revs.update(new_masters)
+
+    # validate masters based on git log --first-parent
+    lines = getpipeoutput(
+        ['git log --first-parent --pretty="%T|%H|%at|%ai|%aN|%aE|%P|%s"',
+         'grep -v ^commit']).split('\n')
+    for line in lines:
+        line = line.strip()
+        if line:
+            graph.add_revision_to_graph(*get_revision_from_line(line, prog), is_master=True)
+
+    # update master branch as appropriate
+    for rev in graph.master_revs:
+        if not graph.revisions[rev].master_parent:
+            parents = graph.linkage[rev]
+            if len(parents) == 1 and parents[0]:
+                graph.revisions[rev].master_parent = parents[0]
+            else:
+                if parents[0]:
+                    logging.warning(f"{rev} has no master parent info. {parents}")
+
+    return graph
+
+
+def get_revision_from_line(line, prog):
+    tree_hash, sha, stamp, time, author, mail, parents, comments = line.split('|', 7)
+    try:
+        stamp = int(stamp)
+    except ValueError:
+        stamp = 0
+    timezone = time.split(' ')[2]
+    domain = '?'
+    if mail.find('@') != -1:
+        domain = mail.rsplit('@', 1)[1]
+    parents = parents.split(' ')
+    revision = Revision(sha, stamp, timezone, author, mail, domain, comments)
+    match = prog.search(comments)
+    if match:
+        revision.master_pr = int(match.group(1))
+    return revision, parents
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    graphs: Dict[str, RevisionGraph] = {}
+    for path in paths:
+        repo_name = os.path.split(path)[1]
+        with (cd.cd(path)):
+            graphs[repo_name] = gen_revision_graph()
+    for k, v in graphs.items():
+        print(f"{k}: {len(v.revisions)} revisions, {len(v.master_revs)} revisions on master")

gitstats/git_csv_generator.py

@@ -1,17 +1,18 @@
 #! /usr/bin/env python3
 import csv
-import glob
 import logging
 import os
 import sys
 
 import multiprocessing_logging
+
+from collections import defaultdict
+
 from gitstats.cd import cd
 
 from gitstats import cli
-from gitstats.data import AuthorTotals, AuthorRow, File, LocByDate, PullRequest, Revision, Tag
-from gitstats.data_generators import gen_author_data, gen_author_totals_data, gen_tag_data, gen_revision_data, \
-    gen_file_data, gen_loc_data, gen_pr_data
+from gitstats.data import PullRequest, Revision
+from gitstats.data_generators import gen_pr_data, gen_revision_graph, gen_complete_file_info
 
 exectime_internal = 0.0
 exectime_external = 0.0
@@ -19,48 +20,39 @@ exectime_external = 0.0
 
 class _FileHandles:
     def __init__(self, output_dir):
-        self.author_info = open(os.path.join(output_dir, 'authors.csv'), 'w', encoding='utf8')
-        self.author_info_writer = csv.writer(self.author_info)
-        self.author_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'Author', 'FilesChanged', 'LinesInserted',
-                                          'LinesDeleted'])
-
         self.author_totals_info = open(os.path.join(output_dir, 'author_totals.csv'), 'w', encoding='utf8')
         self.author_totals_info_writer = csv.writer(self.author_totals_info)
         self.author_totals_info_writer.writerow(["Repo", "Author", "Commits"])
 
-        self.tag_info = open(os.path.join(output_dir, 'tags.csv'), 'w', encoding='utf8')
-        self.tag_info_writer = csv.writer(self.tag_info)
-        self.tag_info_writer.writerow(["Repo", "CommitHash", "Timestamp", "TotalCommits", "Author", "AuthorCommits"])
-
         self.revision_info = open(os.path.join(output_dir, 'revs.csv'), 'w', encoding='utf8')
         self.revision_info_writer = csv.writer(self.revision_info)
         self.revision_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'TimeZone', 'Author', 'AuthorEmail',
-                                            'Domain', 'FilesChanged'])
-
-        self.files_info = open(os.path.join(output_dir, 'files.csv'), 'w', encoding='utf8')
-        self.files_info_writer = csv.writer(self.files_info)
-        self.files_info_writer.writerow(['Repo', 'File', 'Ext', 'Size', 'Lines', 'Resource'])
+                                            'Domain'])
 
         self.loc_info = open(os.path.join(output_dir, 'loc.csv'), 'w', encoding='utf8')
         self.loc_info_writer = csv.writer(self.loc_info)
-        self.loc_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'FileCount', 'LinesInserted', 'LinesDeleted',
-                                       'TotalLines'])
+        self.loc_info_writer.writerow(['repo', 'hash', 'stamp', 'language', 'files', 'lines', 'code', 'comments',
+                                       'blanks'])
+
+        self.loc_delta = open(os.path.join(output_dir, 'loc_delta.csv'), 'w', encoding='utf8')
+        self.loc_delta_writer = csv.writer(self.loc_delta)
+        self.loc_delta_writer.writerow(['repo', 'hash', 'stamp', 'author', 'language', 'files', 'lines', 'code',
+                                        'comments', 'blanks'])
 
         self.repo_info = open(os.path.join(output_dir, 'repo.csv'), 'w', encoding='utf8')
         self.repo_info_writer = csv.writer(self.repo_info)
-        self.repo_info_writer.writerow(['Repo', 'TotalFiles', 'TotalLines'])
+        self.repo_info_writer.writerow(['Repo', 'Language', 'TotalFiles', 'TotalLines', 'TotalCodeLines', 'TotalCommentLlines',
+                                        'TotalBlankLines'])
 
         self.prs_info = open(os.path.join(output_dir, 'prs.csv'), 'w', encoding='utf8')
         self.prs_info_writer = csv.writer(self.prs_info)
         self.prs_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'ParentHashMaster', 'ParentHashBranch', 'PrMergeDuration'])
 
     def close(self):
-        self.author_info.close()
         self.author_totals_info.close()
-        self.tag_info.close()
         self.revision_info.close()
-        self.files_info.close()
         self.loc_info.close()
+        self.loc_delta.close()
         self.repo_info.close()
         self.prs_info.close()
 
@@ -69,8 +61,7 @@ class GitCsvGenerator():
         self.conf = conf
         self.files: _FileHandles = None
         self.output_dir = output_dir
-        self.resource_files = []
-        self.igore_files = ''
+        self.begin, self.end = cli.get_begin_end_timestamps(conf)
 
     def __enter__(self):
         self.files = _FileHandles(self.output_dir)
@@ -81,72 +72,109 @@ class GitCsvGenerator():
     def collect(self, dir):
 
         with cd(dir):
-            self.resource_files = [file for file in glob.glob(self.conf['resrouce_file_pattern'], recursive=True) if os.path.isfile(file)]
-
-            if self.resource_files:
-                self.ignore_files = '" "'.join([f":(exclude){file}" for file in self.resource_files])
-                self.ignore_files = f'-- "{self.ignore_files}"'
-
             if len(self.conf['project_name']) == 0:
                 self.projectname = os.path.basename(os.path.abspath(dir))
             else:
                 self.projectname = self.conf['project_name']
 
-            self.get_total_authors()
-            self.get_tags()
-            self.get_revision_info()
-            self.get_file_info()
-            self.get_loc_info()
-            self.get_author_info()
-            self.get_pr_info()
+            graph = gen_revision_graph()
+            gen_complete_file_info(graph)
+
+            self.extract_total_authors(graph)
+            self.extract_pr_info(graph)
+            self.extract_code_info(graph)
+            self.extract_revision_info(graph)
+            # self.get_revision_info(graph)
+            # self.get_tags()
+            # self.get_file_info()
+            # self.get_loc_info()
+            # self.get_author_info()
 
-    def get_total_authors(self):
+    def extract_total_authors(self, graph):
         logging.info(f"Getting author totals for {self.projectname}")
-        def row_processor(row: AuthorTotals):
-            self.files.author_totals_info_writer.writerow([self.projectname, row.author, row.total_commits])
-        gen_author_totals_data(self.conf, row_processor)
-
-    def get_tags(self):
-        logging.info(f"Getting tag info for {self.projectname}")
-        def row_processor(row: Tag):
-            for author, commits in row.authors.items():
-                self.files.tag_info_writer.writerow([self.projectname, row.hash, row.stamp, row.commits, author, commits])
-        gen_tag_data(self.conf, row_processor)
-
-    def get_revision_info(self):
-        logging.info(f"Getting rev info for {self.projectname}")
-        def row_processor(row: Revision):
-            self.files.revision_info_writer.writerow([self.projectname, row.hash, row.stamp, row.timezone, row.author,
-                                                      row.email, row.domain, row.file_count])
-        gen_revision_data(self.conf, row_processor)
-
-    def get_file_info(self):
-        logging.info(f"Getting file info for {self.projectname}")
-        def row_processor(row: File):
-            self.files.files_info_writer.writerow([self.projectname, row.full_path, row.ext, row.size, row.lines, row.full_path in self.resource_files])
-        gen_file_data(self.conf, row_processor)
-
-    def get_loc_info(self):
-        logging.info(f"Getting LOC info for {self.projectname}")
-        def row_processor(row: LocByDate):
-            self.files.loc_info_writer.writerow([self.projectname, row.hash, row.stamp, row.file_count,
-                                                 row.lines_inserted, row.lines_deleted, row.total_lines])
-        total_files, total_lines = gen_loc_data(self.conf, row_processor, self.ignore_files)
-        self.files.repo_info_writer.writerow([self.projectname, total_files, total_lines])
-
-    def get_author_info(self):
-        logging.info(f"Getting author info for {self.projectname}")
-        def row_processor(row: AuthorRow):
-            self.files.author_info_writer.writerow([self.projectname, row.hash, row.stamp, row.author,
-                                                    row.files_modified, row.lines_inserted, row.lines_deleted])
-        gen_author_data(self.conf, row_processor, self.ignore_files)
-
-    def get_pr_info(self):
+
+        authors = defaultdict(int)
+        for rev in graph.revisions.values():
+            # don't include merge to master as a commit in counting total author
+            # commits.
+            if rev.stamp >= self.begin and rev.stamp <= self.end and rev.master_pr == 0:
+                authors[rev.author] += 1
+
+        for author, total_commits in authors.items():
+            self.files.author_totals_info_writer.writerow([self.projectname, author, total_commits])
+
+    def extract_pr_info(self, graph):
         logging.info(f"Getting pull request info for {self.projectname}")
         def row_processor(row: PullRequest):
-            self.files.prs_info_writer.writerow([self.projectname, row.hash, row.stamp, row.master_rev,
-                                                    row.branch_rev, row.duration.total_seconds()])
-        gen_pr_data(self.conf, row_processor)
+            if row.stamp >= self.begin and row.stamp <= self.end:
+                self.files.prs_info_writer.writerow([self.projectname, row.hash, row.stamp, row.master_rev,
+                                                        row.branch_rev, row.duration.total_seconds()])
+        gen_pr_data(row_processor, graph)
+
+    def extract_code_info(self, graph):
+        rev_max: Revision = None
+        for rev in graph.master_revs:
+            revision: Revision = graph.revisions[rev]
+            if not rev_max or revision.stamp > rev_max.stamp:
+                rev_max = revision
+            if revision.stamp >= self.begin and revision.stamp <= self.end:
+                for lang, file_info in revision.delta.items():
+                        if file_info.file_count or \
+                                file_info.line_count or \
+                                file_info.code_line_count or \
+                                file_info.comment_line_count or \
+                                file_info.blank_line_count:
+
+                            if revision.branch_parent in graph.revisions:
+                                parent = revision.branch_parent
+                            else:
+                                parent = revision.master_parent
+                            if parent:
+                                self.files.loc_delta_writer.writerow([self.projectname,
+                                                 revision.hash,
+                                                 revision.stamp,
+                                                 graph.revisions[parent].author,
+                                                 lang,
+                                                 file_info.file_count,
+                                                 file_info.line_count,
+                                                 file_info.code_line_count,
+                                                 file_info.comment_line_count,
+                                                 file_info.blank_line_count])
+                for lang, file_info in revision.file_infos.items():
+                        if file_info.file_count or \
+                                file_info.line_count or \
+                                file_info.code_line_count or \
+                                file_info.comment_line_count or \
+                                file_info.blank_line_count:
+                            self.files.loc_info_writer.writerow([self.projectname,
+                                             revision.hash,
+                                             revision.stamp,
+                                             lang,
+                                             file_info.file_count,
+                                             file_info.line_count,
+                                             file_info.code_line_count,
+                                             file_info.comment_line_count,
+                                             file_info.blank_line_count])
+
+        for file_info in rev_max.file_infos.values():
+            self.files.repo_info_writer.writerow([self.projectname,
+                                                  file_info.language,
+                                                  file_info.file_count,
+                                                  file_info.line_count,
+                                                  file_info.code_line_count,
+                                                  file_info.comment_line_count,
+                                                  file_info.blank_line_count])
+
+    def extract_revision_info(self, graph):
+        for revision in graph.revisions.values():
+            if revision.stamp >= self.begin and revision.stamp <= self.end:
+                self.files.revision_info_writer.writerow([self.projectname,
+                                                          revision.hash,
+                                                          revision.stamp,
+                                                          revision.timezone,
+                                                          revision.author,
+                                                          revision.email,
+                                                          revision.domain])
 
 def gen_csv():
     conf, paths, outputpath = cli.get_cli()