Fix issue #6: eliminate any hard-code in Deduplicator and Warcrefs.

MohammedElsayyed · MohammedElsayyed · commit 7591d3d6bbdb · 2015-08-05T15:58:46.000+02:00
diff --git a/src/main/java/org/bibalex/warcrefs/Deduplicator.java b/src/main/java/org/bibalex/warcrefs/Deduplicator.java
@@ -27,15 +27,19 @@ public class Deduplicator
 {
     private List<String> warcFiles = new ArrayList();
     private List<String> warcRecords = new ArrayList();
-    private RandomAccessFile digests;       
-    // Default length (8k) that will be copied (written) to dedup_warc
-    private int bufferSize = 8129;
+    private RandomAccessFile digests;
+    private int bufferSize;
+    private String currentWarc;     // represents absolute path of the current warc which is under processing
     
-    public Deduplicator() { }
+    public Deduplicator() 
+    {
+        // Default length (8k) that will be copied (written) to dedup_warc
+        bufferSize = 8129;
+    }
     
     public Deduplicator( int size )
     {
-        bufferSize = size;
+        setBufferSize( size );
     }
     
     public void deduplicate( String digestsPath, String rootDirPath )
@@ -49,7 +53,8 @@ public void deduplicate( String digestsPath, String rootDirPath )
         findAllWarcsRecursively( rootDir );
         
         for ( String warc : warcFiles )
-        {   
+        {
+            currentWarc = warc;
             writeWarcDedup( warc );
             printWarcRecords( warcRecords );
         }
@@ -87,9 +92,6 @@ private void writeWarcDedup( String warcAbsolutePath ) throws IOException
                 FileOutputStream warcOutputStream = new FileOutputStream( 
                         warcDedupAbsolutePath );
                 
-                // Delete the following line after testing
-                //FileOutputStream warcOutputStream2 = new FileOutputStream( "/home/msm/warcrefs/warc.gz" ); // "/home/msm/warcrefs/warc.warc.gz"
-
                 int preOffset = 0;
                 int preLength = 0;
                 int offset = 0;
@@ -164,9 +166,6 @@ private void writeWarcDedup( String warcAbsolutePath ) throws IOException
                 warcInputStream.close();
                 //warcRevisitInputStream.close();
                 warcOutputStream.close();
-                
-                // Delete the following line after testing
-                //warcOutputStream2.close();
             }
             catch ( Exception e )
             {
@@ -182,7 +181,7 @@ private void writeRevisitRecord( FileInputStream fis, FileOutputStream fos,
         WarcReader wrc = new WarcReaderCompressed();
         
         // TODO: should not create a new FileInputStream object when invoking writeRevisitRecord method
-        fis = new FileInputStream( "/home/msm/warcrefs/JAN25_00336-20110731050545553-00076-14438~ia714237.archive.bibalex.org~8443.warc.gz" );
+        fis = new FileInputStream( currentWarc );
         fis.skip( offset );
         WarcRecord record = wrc.getNextRecordFrom( fis, offset );
         
diff --git a/src/main/java/org/bibalex/warcrefs/Warcrefs.java b/src/main/java/org/bibalex/warcrefs/Warcrefs.java
@@ -11,9 +11,9 @@ public class Warcrefs
 {
     public static void main( String[] args ) throws IOException
     {
-        int bufferSize = 8129;                               // args[0]
-        String digestsPath = "/home/msm/warcrefs/digests";   // args[1]
-        String rootDirPath = "/home/msm/warcrefs";           // args[2]
+        int bufferSize = Integer.parseInt( args[ 0 ] ); // 8129
+        String digestsPath = args[ 1 ];                 // path to digests file
+        String rootDirPath = args[ 2 ];                 // warcs directory
         
         try
         {

Original file line number	Diff line number	Diff line change
`@@ -11,9 +11,9 @@ public class Warcrefs`
`11`	`11`	`{`
`12`	`12`	`public static void main( String[] args ) throws IOException`
`13`	`13`	`{`
`14`		`- int bufferSize = 8129; // args[0]`
`15`		`- String digestsPath = "/home/msm/warcrefs/digests"; // args[1]`
`16`		`- String rootDirPath = "/home/msm/warcrefs"; // args[2]`
	`14`	`+ int bufferSize = Integer.parseInt( args[ 0 ] ); // 8129`
	`15`	`+ String digestsPath = args[ 1 ]; // path to digests file`
	`16`	`+ String rootDirPath = args[ 2 ]; // warcs directory`
`17`	`17`
`18`	`18`	`try`
`19`	`19`	`{`