Improve and expose CLI arguments for clone detection.

Author: Miltos Allamanis

DuplicateCodeDetector/CloneDetector.cs

@@ -1,7 +1,9 @@
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.IO.Compression;
 using System.Linq;
@@ -13,16 +15,20 @@ namespace NearCloneDetector
 {
     class CloneDetector
     {
-        public struct TokenData
-        {
-            public string filename;
-            public string[] tokens;
-        }
-
         private readonly Dictionary<string, Dictionary<string, SparseVector>> _index = new Dictionary<string, Dictionary<string, SparseVector>>();
         public int NumFiles => _index.Sum(prj => prj.Value.Count);
         private readonly FeatureDictionary _dict = new FeatureDictionary();
 
+        private readonly string _tokensFieldName;
+        private readonly string[] _identifyingFields;
+
+        public CloneDetector(string tokensField, string[] entryIdFields)
+        {
+            _tokensFieldName = tokensField;
+            Debug.Assert(entryIdFields.Length > 0);
+            _identifyingFields = entryIdFields;
+        }
+
         public readonly Dictionary<string, HashSet<string>> Duplicates = new Dictionary<string, HashSet<string>>();
 
         [MethodImpl(MethodImplOptions.Synchronized)]
@@ -52,11 +58,17 @@ private void AddDuplicate(string file1, string file2)
 
         public void BuildIndexForProjects(string tokenizedFilesPath)
         {
-            var allFiles = Directory.GetFiles(tokenizedFilesPath, "*.jsonl.gz");
+            var allFiles = Directory.GetFiles(tokenizedFilesPath, "*.gz")
+                .Select(f=> Path.Combine(tokenizedFilesPath, f));
+            BuildIndexFromFiles(allFiles);
+        }
+
+        public void BuildIndexFromFiles(IEnumerable<string> allFiles)
+        {
             foreach (var projectDir in allFiles)
             {
                 Console.WriteLine($"Indexing project {projectDir}");
-                BuildIndexForProject(Path.Combine(tokenizedFilesPath, projectDir));
+                BuildIndexForProject(projectDir);
             }
         }
 
@@ -77,14 +89,15 @@ public void BuildIndexForProject(string parsedJsonlPath)
                         line = text.ReadLine();
                         continue;
                     }
-                    var tokenData = JsonConvert.DeserializeObject<TokenData>(line);
-                    var tokenCounter = Count(tokenData.tokens);
+                    var tokenData = JsonConvert.DeserializeObject<IDictionary<string, object>>(line);
+                    var tokenCounter = Count(((JArray)tokenData[_tokensFieldName]).Select(t=>t.ToString()));
 
                     if (tokenCounter.Sum(tc => tc.Count) >= MIN_NUM_TOKENS_FOR_FILE)
                     {
                         var spVect = new SparseVector();
                         spVect.AddElements(tokenCounter.Select(tc => (_dict.AddOrGet(tc.Token), tc.Count)));
-                        projectIndex[tokenData.filename] = spVect;
+                        var entryIdentifier = string.Join(":", _identifyingFields.Select(idf => tokenData[idf].ToString()));
+                        projectIndex[entryIdentifier] = spVect;
                     }
                     line = text.ReadLine();
                 }

DuplicateCodeDetector/CloneDetectorCli.cs

@@ -1,45 +1,60 @@
-using System;
+using DocoptNet;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 
 namespace NearCloneDetector
 {
     class CloneDetectorCli
     {
-        static void Main(string[] args)
+        private const string usage = @"Near Clone Detector.
+
+    Usage:
+      CloneDetectorCli [options] (--dir=<folder> | --input=<file>) <output-file-prefix>
+
+    Options:
+      -h --help                      Show this screen.
+      --dir=<path>                   Directory where .jsonl.gz files live.
+      --input=<path>                 The path to the input .jsonl.gz file.
+      --id-fields=<fields>           A colon (:)-separated list of names of fields that form the identity of each entry [default: filename].
+      --tokens-field=<name>          The name of the field containing the tokens of the code [default: tokens].
+      --key-jaccard-threshold=<val>  The Jaccard similarity threshold for token-sets [default: 0.8].
+      --jaccard-threshold=<val>      The Jaccard similarity threshold for token multisets [default: 0.7].
+
+    ";
+
+            static void Main(string[] args)
+        {
+            var arguments = new Docopt().Apply(usage, args, version: "Near Clone Detector", exit: true);
+            DetectClones(arguments);            
+        }
+
+
+
+        public static void DetectClones(IDictionary<string, ValueObject> arguments)
         {
-            if (args.Length != 3)
+            var cd = new CloneDetector(arguments["--tokens-field"].ToString(), arguments["--id-fields"].ToString().Split(':'));
+            if (arguments.TryGetValue("--dir", out var dataDirectory) && dataDirectory != null)
             {
-                Console.WriteLine("Usage detect <rootDir> <clonesOutputFilePrefix>");
-                return;
+                cd.BuildIndexForProjects(dataDirectory.ToString());
             }
-            var rootDir = args[1];
-            var clonesOutputFilePrefix = args[2];
-
-            if (args[0] == "detect")
+            else if (arguments.TryGetValue("--input", out var dataFile) && dataFile != null)
             {
-                DetectClones(rootDir, clonesOutputFilePrefix);
+                cd.BuildIndexFromFiles(new[] { dataFile.ToString() });
             }
             else
             {
-                throw new NotSupportedException($"Unsupported option {args[0]}");
+                throw new Exception("Either --dir or --input need to be provided.");
             }
-        }
-
-
-
-        public static void DetectClones(string rootDir, string clonesOutputFilePrefix,
-            double keyJaccardSimilarityThreshold = 0.8, double jaccardSimilarityThreshold = 0.7)
-        {
-            var cd = new CloneDetector();
-            cd.BuildIndexForProjects(rootDir);
-
+                
             Console.WriteLine($"[{DateTime.Now}] Searching for near duplicates...");
             var startTime = DateTime.Now;
-            using (var writer = new StreamWriter(clonesOutputFilePrefix + ".txt"))
+            using (var writer = new StreamWriter(arguments["<output-file-prefix>"].ToString() + ".log"))
             {
                 foreach (var (File1, File2, JaccardSimilarity, KeyJacardSimilarity) in
-                    cd.FindNearDuplicates(keyJaccardSimilarityThreshold, jaccardSimilarityThreshold))
+                    cd.FindNearDuplicates(double.Parse(arguments["--key-jaccard-threshold"].ToString()),
+                                          double.Parse(arguments["--jaccard-threshold"].ToString())))
                 {
                     Console.WriteLine($"Near duplicate: ({File1}-->{File2}) [scores: {JaccardSimilarity: #.##}, {KeyJacardSimilarity: #.#}]");
                     writer.WriteLine($"{File1},{File2},{JaccardSimilarity},{KeyJacardSimilarity}");
@@ -51,7 +66,7 @@ public static void DetectClones(string rootDir, string clonesOutputFilePrefix,
             Console.WriteLine($"Duplicate search took {elapsedTime}.");
 
             var cloneGroups = new CloneGroups(cd.Duplicates.SelectMany(c => c.Value.Select(f => (c.Key, f))));
-            cloneGroups.SaveToJson(clonesOutputFilePrefix + ".json");
+            cloneGroups.SaveToJson(arguments["<output-file-prefix>"].ToString() + ".json");
         }
     }
 }

DuplicateCodeDetector/DuplicateCodeDetector.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="docopt.net" Version="0.6.1.10" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
   </ItemGroup>
 

README.md

@@ -6,9 +6,9 @@ This cross-platform sample tool detects exact and near duplicates of code mainta
 
 To run the near-duplicate detection run:
 ```
-$ dotnet run /path/to/DuplicateCodeDetector.csproj detect path/to/dataFolder outputFile
+$ dotnet run /path/to/DuplicateCodeDetector.csproj [options] --dir=<folder> <output-file-prefix>
 ```
-This will use all the `.jsonl.gz` files in the `dataFolder` and output an `outputFile` with the duplicate pairs and an `outputFile.json` with the groups of detected duplicates.
+This will use all the `.gz` files in the `<folder>` and output an `<output-file-prefix>.json` with the groups of detected duplicates. Invoke `--help` for more options.
 
 ### Input Data
 
@@ -19,6 +19,7 @@ The input data should be one or more `.jsonl.gz` files. These are compressed [JS
     "tokens" : ["list", "of", "tokens", "in", "file"]
 }
 ```
+Alternative formats can be accepted by providing the `--tokens-field` and `--id-fields` options.
 
 The `tokenizers` folder in this repository contains tokenizers for 
 C\#, Java, JavaScript and Python. Please, feel free to contribute tokenizers for other languages too.