Create csv generator as a parallel to HTML Report Creator

gitstats/cli.py

@@ -25,7 +25,6 @@ Usage: gitstats [options] <gitpath..> <outputpath>
 
 Options:
 -c key=value     Override configuration value
--n key=value     Define author name equivalency (key will treated the same as value)
 
 Default config values:
 {conf}

gitstats/data/__init__.py

@@ -4,4 +4,5 @@ from .author_totals import AuthorTotals
 from .tag import Tag
 from .revision import Revision
 from .file import File
-from .loc_by_date import LocByDate
+from .loc_by_date import LocByDate
+from .pr import PullRequest

gitstats/data/author_row.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 
 @dataclass
 class AuthorRow:
-    sha: str
+    hash: str
     stamp: int
     author: str
     files_modified: int

gitstats/data/revision.py

@@ -6,7 +6,7 @@ from typing import Dict
 
 @dataclass
 class Revision:
-    sha: str
+    hash: str
     stamp: int
     timezone: int = 0
     author: str = ''

gitstats/data_generators/__init__.py

@@ -3,4 +3,5 @@ from .gen_author_totals import gen_author_totals_data
 from .gen_tag_data import gen_tag_data
 from .gen_revision_data import gen_revision_data
 from .gen_file_data import gen_file_data
-from .gen_loc_data import gen_loc_data
+from .gen_loc_data import gen_loc_data
+from .gen_pr_data import gen_pr_data

gitstats/data_generators/gen_author_data.py

@@ -81,8 +81,7 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-
-                gen_author_data(
-                    conf,
-                    lambda row: writer.writerow([repo_name, row.sha, row.stamp, row.author, row.files_modified,
-                                                 row.lines_inserted, row.lines_deleted]))
+                def row_processor(row: AuthorRow):
+                    writer.writerow([repo_name, row.hash, row.stamp, row.author, row.files_modified,
+                                     row.lines_inserted, row.lines_deleted])
+                gen_author_data(conf, row_processor)

gitstats/data_generators/gen_author_totals.py

@@ -42,6 +42,6 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-                gen_author_totals_data(
-                    conf,
-                    lambda row: writer.writerow([repo_name, row.author, row.total_commits]))
+                def row_processor(row: AuthorTotals):
+                    writer.writerow([repo_name, row.author, row.total_commits])
+                gen_author_totals_data(conf, row_processor)

gitstats/data_generators/gen_file_data.py

@@ -59,7 +59,6 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-
-                gen_file_data(
-                    conf,
-                    lambda row: writer.writerow([repo_name, row.full_path, row.ext, row.size, row.lines]))
+                def row_processor(row: File):
+                    writer.writerow([repo_name, row.full_path, row.ext, row.size, row.lines])
+                gen_file_data(conf, row_processor)

gitstats/data_generators/gen_loc_data.py

@@ -7,6 +7,8 @@ from gitstats import cli, cd
 from gitstats.miscfuncs import getlogrange, getpipeoutput, getstatsummarycounts
 from gitstats.data import LocByDate
 
+# TODO: the author isn't working here because it's the commit that merges to master, so we
+# TODO: probably need to back up a commit. Each commit here represents a merged PR
 
 def gen_loc_data(conf, row_processor):
     '''
@@ -74,7 +76,7 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-
-                gen_loc_data(
-                    conf,
-                    lambda row: writer.writerow([repo_name, row.hash, row.stamp, row.file_count, row.lines_inserted, row.lines_deleted, row.total_lines]))
+                def row_processor(row: LocByDate):
+                    writer.writerow([repo_name, row.hash, row.stamp, row.file_count, row.lines_inserted,
+                                     row.lines_deleted, row.total_lines])
+                gen_loc_data(conf, row_processor)

gitstats/data_generators/gen_revision_data.py

@@ -68,8 +68,7 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-
-                gen_revision_data(
-                    conf,
-                    lambda row: writer.writerow([repo_name, row.sha, row.stamp, row.timezone, row.author, row.email,
-                                                 row.domain, row.file_count]))
+                def row_processor(row: Revision):
+                    writer.writerow([repo_name, row.hash, row.stamp, row.timezone, row.author, row.email,
+                                     row.domain, row.file_count])
+                gen_revision_data(conf, row_processor)

gitstats/data_generators/gen_tag_data.py

@@ -66,9 +66,7 @@ if __name__ == "__main__":
         for path in paths:
             repo_name = os.path.split(path)[1]
             with (cd.cd(path)):
-
-                def process_row(row):
+                def row_processor(row: Tag):
                     for author, commits in row.authors.items():
                         writer.writerow([repo_name, row.hash, row.stamp, row.commits, author, commits])
-
-                gen_tag_data(conf, process_row)
+                gen_tag_data(conf, row_processor)

gitstats/git_csv_generator.py

@@ -0,0 +1,149 @@
+import csv
+import logging
+import os
+import sys
+
+import multiprocessing_logging
+
+from gitstats import cli
+from gitstats.data import AuthorTotals, AuthorRow, File, LocByDate, Revision, Tag
+from gitstats.data_generators import gen_author_data, gen_author_totals_data, gen_tag_data, gen_revision_data, \
+    gen_file_data, gen_loc_data
+
+exectime_internal = 0.0
+exectime_external = 0.0
+
+
+class _FileHandles:
+    def __init__(self, output_dir):
+        self.author_info = open(os.path.join(output_dir, 'authors.csv'), 'w', encoding='utf8')
+        self.author_info_writer = csv.writer(self.author_info)
+        self.author_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'Author', 'FilesChanged', 'LinesInserted',
+                                          'LinesDeleted'])
+
+        self.author_totals_info = open(os.path.join(output_dir, 'author_totals.csv'), 'w', encoding='utf8')
+        self.author_totals_info_writer = csv.writer(self.author_totals_info)
+        self.author_totals_info_writer.writerow(["Repo", "Author", "Commits"])
+
+        self.tag_info = open(os.path.join(output_dir, 'tags.csv'), 'w', encoding='utf8')
+        self.tag_info_writer = csv.writer(self.tag_info)
+        self.tag_info_writer.writerow(["Repo", "CommitHash", "Timestamp", "TotalCommits", "Author", "AuthorCommits"])
+
+        self.revision_info = open(os.path.join(output_dir, 'revs.csv'), 'w', encoding='utf8')
+        self.revision_info_writer = csv.writer(self.revision_info)
+        self.revision_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'TimeZone', 'Author', 'AuthorEmail',
+                                            'Domain', 'FilesChanged'])
+
+        self.files_info = open(os.path.join(output_dir, 'files.csv'), 'w', encoding='utf8')
+        self.files_info_writer = csv.writer(self.files_info)
+        self.files_info_writer.writerow(['Repo', 'File', 'Ext', 'Size', 'Lines'])
+
+        self.loc_info = open(os.path.join(output_dir, 'loc.csv'), 'w', encoding='utf8')
+        self.loc_info_writer = csv.writer(self.loc_info)
+        self.loc_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'FileCount', 'LinesInserted', 'LinesDeleted',
+                                       'TotalLines'])
+
+    def close(self):
+        self.author_info.close()
+        self.author_totals_info.close()
+        self.tag_info.close()
+        self.revision_info.close()
+        self.files_info.close()
+        self.loc_info.close()
+
+
+class GitCsvGenerator():
+    def __init__(self, conf, output_dir):
+        self.conf = conf
+        self.files: _FileHandles = None
+        self.output_dir = output_dir
+
+    def __enter__(self):
+        self.files = _FileHandles(self.output_dir)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.files.close()
+
+    def collect(self, dir):
+        if len(self.conf['project_name']) == 0:
+            self.projectname = os.path.basename(os.path.abspath(dir))
+        else:
+            self.projectname = self.conf['project_name']
+
+        self.get_total_authors()
+        self.get_tags()
+        self.get_revision_info()
+        self.get_file_info()
+        self.get_loc_info()
+        self.get_author_info()
+
+    def get_total_authors(self):
+        logging.info(f"Getting author totals for {self.projectname}")
+        def row_processor(row: AuthorTotals):
+            self.files.author_totals_info_writer.writerow([self.projectname, row.author, row.total_commits])
+        gen_author_totals_data(self.conf, row_processor)
+
+    def get_tags(self):
+        logging.info(f"Getting tag info for {self.projectname}")
+        def row_processor(row: Tag):
+            for author, commits in row.authors.items():
+                self.files.tag_info_writer.writerow([self.projectname, row.hash, row.stamp, row.commits, author, commits])
+        gen_tag_data(self.conf, row_processor)
+
+    def get_revision_info(self):
+        logging.info(f"Getting rev info for {self.projectname}")
+        def row_processor(row: Revision):
+            self.files.revision_info_writer.writerow([self.projectname, row.hash, row.stamp, row.timezone, row.author,
+                                                      row.email, row.domain, row.file_count])
+        gen_revision_data(self.conf, row_processor)
+
+    def get_file_info(self):
+        logging.info(f"Getting file info for {self.projectname}")
+        def row_processor(row: File):
+            self.files.files_info_writer.writerow([self.projectname, row.full_path, row.ext, row.size, row.lines])
+        gen_file_data(self.conf, row_processor)
+
+    def get_loc_info(self):
+        logging.info(f"Getting LOC info for {self.projectname}")
+        def row_processor(row: LocByDate):
+            self.files.loc_info_writer.writerow([self.projectname, row.hash, row.stamp, row.file_count,
+                                                 row.lines_inserted, row.lines_deleted, row.total_lines])
+        gen_loc_data(self.conf, row_processor)
+
+    def get_author_info(self):
+        logging.info(f"Getting author info for {self.projectname}")
+        def row_processor(row: AuthorRow):
+            self.files.author_info_writer.writerow([self.projectname, row.hash, row.stamp, row.author,
+                                                    row.files_modified, row.lines_inserted, row.lines_deleted])
+        gen_author_data(self.conf, row_processor)
+
+def gen_csv():
+    conf, paths, outputpath = cli.get_cli()
+
+    logging.basicConfig(level=conf['logging'], format='%(message)s')
+    multiprocessing_logging.install_mp_handler()
+    try:
+        os.makedirs(outputpath)
+    except OSError:
+        pass
+    if not os.path.isdir(outputpath):
+        logging.fatal('Output path is not a directory or does not exist')
+        sys.exit(1)
+
+    logging.info(f'Output path: {outputpath}')
+
+    data = GitCsvGenerator(conf, outputpath)
+    with data:
+        for gitpath in paths:
+            logging.info(f'Git path: {gitpath}')
+
+            prevdir = os.getcwd()
+            os.chdir(gitpath)
+
+            logging.info('Collecting data...')
+            data.collect(gitpath)
+
+            os.chdir(prevdir)
+
+if __name__ == '__main__':
+    gen_csv()

gitstats/gitdatacollector.py

@@ -15,7 +15,6 @@ class GitDataCollector(DataCollector):
         
     def collect(self, directory):
         super(GitDataCollector, self).collect(directory)
-
         self.total_authors += self.get_total_authors()
         self.get_tags()
         self.get_revision_info()