Dwr/pull request tracking (#1)

* Begin looking at code to track pull requests (just looking at bitbucket for now, should translate into github and gitlabs)

* Finish PullRequest duration calculation

* Finish PullRequest duration calculation

* Finish PullRequest duration calculation

* use total_seconds duration for PR analysis

gitstats/data/pr.py

@@ -0,0 +1,15 @@
+from datetime import timedelta
+from dataclasses import dataclass
+from typing import List
+
+@dataclass
+class PullRequest:
+    stamp: int
+    hash: str
+    author: str
+    parent_hashes: List[str]
+    branch_rev: str = None
+    master_rev: str = None
+    duration: timedelta = None
+    invalid_pr: bool = False
+

gitstats/data_generators/gen_pr_data.py

@@ -0,0 +1,90 @@
+import csv
+import logging
+import os
+
+from multiprocessing import Pool
+
+from gitstats import cli, cd
+from gitstats.miscfuncs import getlogrange, getpipeoutput, gettimedelta
+from gitstats.data import PullRequest
+
+
+def gen_pr_data(conf, row_processor):
+    '''
+    Given a configuration, pull revision information. For
+    each author, callback to the row_processor passing an PullRequest
+
+    :param conf: configuration (mostly used for date limits)
+    :param row_processor: function to receive the callback
+    :return: None
+    '''
+
+    prs = {} # hash -> PullRequest
+
+    # DBG: git log --all --grep="Merge pull request .* to master" --shortstat --pretty=format:"%H %at %aN" --since="2017-10-01" "HEAD"', 'grep -v ^commit'
+    lines = getpipeoutput(
+        ['git log --all --grep="Merge pull request .* to master" --shortstat '
+         '--pretty=format:"%%H %%at %%aN|%%P" %s' % getlogrange(conf, 'HEAD'),
+         'grep -v ^"files changed"']).split('\n')
+    for line in lines:
+        line = line.strip()
+        if line and not 'files changed' in line:
+            parts = line.split(' ', 2)
+            hash = parts[0]
+            try:
+                stamp = int(parts[1])
+            except ValueError:
+                stamp = 0
+            (author, parent_hashes) = parts[2].split('|')
+            parent_hashes = parent_hashes.split(' ')
+            if len(parent_hashes) == 2:
+                prs[hash] = PullRequest(stamp, hash, author, parent_hashes)
+
+    keys = prs.keys()
+    for pr in prs.values():
+        if pr.parent_hashes[0] in keys:
+            pr.master_rev = pr.parent_hashes[0]
+            if pr.parent_hashes[1] in keys:
+                logging.warning(f"Unexpected branching: {pr}")
+                pr.invalid_pr = True
+            else:
+                pr.branch_rev = pr.parent_hashes[1]
+        else:
+            pr.branch_rev = pr.parent_hashes[0]
+            if pr.parent_hashes[1] in keys:
+                pr.master_rev = pr.parent_hashes[1]
+            else:
+                logging.warning(f"Unexpected branching: {pr}")
+                pr.invalid_pr = True
+
+    prs_to_query = [(pr.hash, pr.stamp, pr.branch_rev) for pr in prs.values() if not pr.invalid_pr]
+
+    # # todo: consider putting in a cache for this. There was one in the original code
+    # # DBG:  git log -n 1 --format=%at "ceb3165b51ae0680724fd71e16a5ff836a0de41e"
+    pool = Pool(processes=conf['processes'])
+    time_deltas = pool.map(gettimedelta, prs_to_query)
+    pool.terminate()
+    pool.join()
+    for (hash, timedelta) in time_deltas:
+        pr = prs[hash]
+        pr.duration = timedelta
+        if pr.duration.total_seconds() < 0:
+            pr.invalid_pr = True
+            logging.warning(f"Unexpected. Negative duration: {pr}")
+        else:
+            row_processor(pr)
+
+
+
+if __name__ == "__main__":
+    conf, paths, outputpath = cli.get_cli()
+    with open(outputpath, 'w', encoding='utf8') as f:
+        writer = csv.writer(f)
+        writer.writerow(['repo', 'hash', 'stamp', 'masterRev', 'branchRev', 'prMergeDuration', 'prMergeDurationHr'])
+
+        for path in paths:
+            repo_name = os.path.split(path)[1]
+            with (cd.cd(path)):
+                def row_processor(row: PullRequest):
+                    writer.writerow([repo_name, row.hash, row.stamp, row.master_rev, row.branch_rev, row.duration.total_seconds(), row.duration])
+                gen_pr_data(conf, row_processor)

gitstats/git_csv_generator.py

@@ -6,9 +6,9 @@ import sys
 import multiprocessing_logging
 
 from gitstats import cli
-from gitstats.data import AuthorTotals, AuthorRow, File, LocByDate, Revision, Tag
+from gitstats.data import AuthorTotals, AuthorRow, File, LocByDate, PullRequest, Revision, Tag
 from gitstats.data_generators import gen_author_data, gen_author_totals_data, gen_tag_data, gen_revision_data, \
-    gen_file_data, gen_loc_data
+    gen_file_data, gen_loc_data, gen_pr_data
 
 exectime_internal = 0.0
 exectime_external = 0.0
@@ -47,6 +47,10 @@ class _FileHandles:
         self.repo_info_writer = csv.writer(self.repo_info)
         self.repo_info_writer.writerow(['Repo', 'TotalFiles', 'TotalLines'])
 
+        self.prs_info = open(os.path.join(output_dir, 'prs.csv'), 'w', encoding='utf8')
+        self.prs_info_writer = csv.writer(self.prs_info)
+        self.prs_info_writer.writerow(['Repo', 'CommitHash', 'TimeStamp', 'ParentHashMaster', 'ParentHashBranch', 'PrMergeDuration'])
+
     def close(self):
         self.author_info.close()
         self.author_totals_info.close()
@@ -55,6 +59,7 @@ class _FileHandles:
         self.files_info.close()
         self.loc_info.close()
         self.repo_info.close()
+        self.prs_info.close()
 
 
 class GitCsvGenerator():
@@ -81,6 +86,7 @@ class GitCsvGenerator():
         self.get_file_info()
         self.get_loc_info()
         self.get_author_info()
+        self.get_pr_info()
 
     def get_total_authors(self):
         logging.info(f"Getting author totals for {self.projectname}")
@@ -123,6 +129,13 @@ class GitCsvGenerator():
                                                     row.files_modified, row.lines_inserted, row.lines_deleted])
         gen_author_data(self.conf, row_processor)
 
+    def get_pr_info(self):
+        logging.info(f"Getting pull request info for {self.projectname}")
+        def row_processor(row: PullRequest):
+            self.files.prs_info_writer.writerow([self.projectname, row.hash, row.stamp, row.master_rev,
+                                                    row.branch_rev, row.duration.total_seconds()])
+        gen_pr_data(self.conf, row_processor)
+
 def gen_csv():
     conf, paths, outputpath = cli.get_cli()
 

gitstats/miscfuncs.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 import os
 import re
@@ -96,4 +97,15 @@ def getnumoflinesinblob(blob_id):
     # DBG: git cat-file blob e4f17a621893811250be96c8ef9c37b5e97a1df7', 'wc -l'
     return blob_id, int(getpipeoutput(['git cat-file blob %s' % blob_id, 'wc -l']).split()[0])
 
+def gettimedelta(sha_tuple):
+    """
+    Get the time delta between the time stamp passed in the tuple ([1]) and the sha of the second rev in the tuple ([2])
+    return the result, keyed by the sha of the first rev in the tuple ([0])
+    """
+    # DBG:  git log -n 1 --format=%at "ceb3165b51ae0680724fd71e16a5ff836a0de41e"'
+    timestamp_branch = int(getpipeoutput([' git log -n 1 --format=%%at "%s"' % sha_tuple[2]]).split('\n')[0])
+    delta = datetime.datetime.utcfromtimestamp(sha_tuple[1]) - datetime.datetime.utcfromtimestamp(timestamp_branch)
+
+    return (sha_tuple[0], delta)
+