Merge pull request #10 from takahi-i/refactoring-code

Refactoring code
takahi-i · Feb 5, 2015 · 7971db9 · 7971db9
2 parents 612973f + c48ffb6
commit 7971db9
Show file tree

Hide file tree

Showing 29 changed files with 104 additions and 218 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,5 @@
 language: java
 
 jdk:
-  - openjdk6
-  - openjdk7
+  - oraclejdk7
+  - oraclejdk8
diff --git a/core/lib/apache-cassandra-0.6.0.jar b/core/lib/apache-cassandra-0.6.0.jar
diff --git a/core/lib/cassandra-javautils.jar b/core/lib/cassandra-javautils.jar
diff --git a/core/lib/clhm-production-1.0.jar b/core/lib/clhm-production-1.0.jar
diff --git a/core/lib/commons-lang-2.4.jar b/core/lib/commons-lang-2.4.jar
diff --git a/core/lib/flexjson-1.7.jar b/core/lib/flexjson-1.7.jar
diff --git a/core/lib/hector-0.6.0-15.jar b/core/lib/hector-0.6.0-15.jar
diff --git a/core/lib/high-scale-lib-1.0.jar b/core/lib/high-scale-lib-1.0.jar
diff --git a/core/lib/libthrift-r917130.jar b/core/lib/libthrift-r917130.jar
diff --git a/core/pom.xml b/core/pom.xml
@@ -21,10 +21,10 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>2.3.1</version>
+        <version>3.2</version>
         <configuration>
-          <source>1.5</source>
-          <target>1.5</target>
+          <source>1.7</source>
+          <target>1.7</target>
         </configuration>
       </plugin>
       <plugin>
@@ -45,4 +45,4 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
-</project>
+</project>
diff --git a/core/src/main/java/org/unigram/likelike/common/FsUtil.java b/core/src/main/java/org/unigram/likelike/common/FsUtil.java
@@ -16,13 +16,13 @@
  */
 package org.unigram.likelike.common;
 
-import java.io.IOException;
-import java.util.logging.Level;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import java.io.IOException;
+import java.util.logging.Level;
+
 /**
  * File utility class. 
  */
@@ -49,7 +49,7 @@ public static boolean checkPath(final Path dir,
            final FileSystem fs)
    throws IOException {
         if (fs.exists(dir)) {
-            logger.log(Level.INFO, "Overiding: " + dir.toString());
+            logger.log(Level.INFO, "Overriding: " + dir.toString());
             return fs.delete(dir, true);
         } else {
             logger.log(Level.FINE, "No such file: " + dir.toString());
@@ -80,16 +80,15 @@ public static boolean checkPath(final Path dir,
     */
    public static void clean(final FileSystem fs, 
        final String... fileNames) throws IOException {
-       
-       for (int i = 0; i < fileNames.length; i++) {
-           Path path = new Path(fileNames[i]);
+
+       for (String fileName : fileNames) {
+           Path path = new Path(fileName);
            if (fs.exists(path)) {
-               logger.log(Level.INFO, 
+               logger.log(Level.INFO,
                        "Removing: " + path.toString());
                fs.delete(path, true);
            }
        }
-       return;
-   }   
+   }
 
 }
diff --git a/core/src/main/java/org/unigram/likelike/common/RelatedUsersWritable.java b/core/src/main/java/org/unigram/likelike/common/RelatedUsersWritable.java
@@ -16,16 +16,16 @@
  */
 package org.unigram.likelike.common;
 
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Writable;
+
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.EOFException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-
 /**
  *
  */
@@ -72,7 +72,6 @@ public List<LongWritable> getRelatedUsers() {
      * @param in input stream
      * @throws IOException -
      */
-    @Override
     public void readFields(final DataInput in) throws IOException {
         try {
             int listSize = in.readInt();
@@ -93,7 +92,6 @@ public void readFields(final DataInput in) throws IOException {
      * @param out output stream
      * @throws IOException -
      */
-    @Override
     public void write(final DataOutput out) throws IOException {
         out.writeInt(this.relatedUsers.size());
         for (LongWritable item : this.relatedUsers) {

diff --git a/core/src/main/java/org/unigram/likelike/feature/FeatureExtraction.java b/core/src/main/java/org/unigram/likelike/feature/FeatureExtraction.java
@@ -1,7 +1,5 @@
 
 package org.unigram.likelike.feature;
-import java.io.IOException;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
@@ -13,13 +11,14 @@
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-
 import org.unigram.likelike.common.FsUtil;
 import org.unigram.likelike.common.LikelikeConstants;
-import org.unigram.likelike.util.IdentityReducer;
-import org.unigram.likelike.util.InverseMapper;
 import org.unigram.likelike.util.AddFeatureMapper;
 import org.unigram.likelike.util.AddFeatureReducer;
+import org.unigram.likelike.util.IdentityReducer;
+import org.unigram.likelike.util.InverseMapper;
+
+import java.io.IOException;
 
 /**
  *
@@ -33,7 +32,6 @@ public class FeatureExtraction  extends Configured
      * @return 0 when succeeded
      * @throws Exception -
      */
-    @Override
     public int run(final String[] args) throws Exception {
         Configuration conf = getConf();
         return this.run(args, conf);   

diff --git a/core/src/main/java/org/unigram/likelike/feature/FeatureExtractionReducer.java b/core/src/main/java/org/unigram/likelike/feature/FeatureExtractionReducer.java
@@ -97,13 +97,12 @@ public int compare(final Object o1, final Object o2){
             Map.Entry obj = (Map.Entry) it.next();
             Long feature = (Long) obj.getKey();
             if (!targetFeatures.containsKey(feature)) {
-            	//System.out.println("target: " + target + "\tfeature: " + feature);
-            	try {
-					this.writer.write((Long) target.get(), feature, context);
-				} catch (Exception e) {
-					// TODO Auto-generated catch block
-					e.printStackTrace();
-				}
+                try {
+                    this.writer.write((Long) target.get(), feature, context);
+                } catch (Exception e) {
+                    // TODO Auto-generated catch block
+                    e.printStackTrace();
+                }
             }
         }
     }
@@ -145,7 +144,7 @@ public final void setup(final Context context) {
                 LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER;
 
         jc.set(LikelikeConstants.CASSANDRA_COLUMNFAMILY_NAME, 
-        		LikelikeConstants.LIKELIKE_CASSANDRA_FEATURE_EXTRACTION_COLUMNFAMILY_NAME);
+                LikelikeConstants.LIKELIKE_CASSANDRA_FEATURE_EXTRACTION_COLUMNFAMILY_NAME);
 
         try {
             writerClassName = 

diff --git a/core/src/main/java/org/unigram/likelike/lsh/GetRecommendationsReducer.java b/core/src/main/java/org/unigram/likelike/lsh/GetRecommendationsReducer.java
@@ -40,7 +40,6 @@
 public class GetRecommendationsReducer extends
         Reducer<LongWritable, Candidate, 
         LongWritable, LongWritable> {
-
     /**
      * reduce. 
      * @param key target
@@ -53,8 +52,7 @@ public void reduce(final LongWritable key,
             final Iterable<Candidate> values,
             final Context context)
             throws IOException, InterruptedException {
-
-        HashMap<Long, Double> candidates 
+        HashMap<Long, Double> candidates
             = new HashMap<Long, Double>();
         for (Candidate cand : values) {
             Long tid = cand.getId().get();
@@ -63,10 +61,8 @@ public void reduce(final LongWritable key,
                weight += 1.0; // not use the size of the cluster
                candidates.put(tid, weight);
             } else {
-                candidates.put(tid, 
-                        new Double(1.0));
+                candidates.put(tid, new Double(1.0));
             }
-
             if (candidates.size() > 50000) { // TODO should be parameterized
                 break;
             }
@@ -109,7 +105,7 @@ public void reduce(final LongWritable key,
      */
     @Override
     public final void setup(final Context context) {
-        Configuration jc = null; 
+        Configuration jc = null;
 
         if (context == null) {
             jc = new Configuration();
@@ -118,11 +114,11 @@ public final void setup(final Context context) {
         }
 
         this.maxOutputSize = jc.getLong(
-                LikelikeConstants.MAX_OUTPUT_SIZE , 
+                LikelikeConstants.MAX_OUTPUT_SIZE,
                 LikelikeConstants.DEFAULT_MAX_OUTPUT_SIZE);
-        
-        this.comparator = new Comparator<Object>(){
-            public int compare(final Object o1, final Object o2){
+
+        this.comparator = new Comparator<Object>() {
+            public int compare(final Object o1, final Object o2) {
                 Map.Entry e1 = (Map.Entry) o1;
                 Map.Entry e2 = (Map.Entry) o2;
                 Double e1Value = (Double) e1.getValue();
@@ -132,28 +128,20 @@ public int compare(final Object o1, final Object o2){
         };
 
         // create writer
-        String writerClassName = 
+        String writerClassName =
                 LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER;
         try {
-            writerClassName = 
+            writerClassName =
                     jc.get(LikelikeConstants.LIKELIKE_OUTPUT_WRITER,
-                    LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER);
+                            LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER);
             Class<? extends IWriter> extractorClass = Class.forName(
                     writerClassName).asSubclass(IWriter.class);
             Constructor<? extends IWriter> constructor = extractorClass
                     .getConstructor(Configuration.class);
             this.writer = constructor.newInstance(jc);
-        } catch (NoSuchMethodException nsme) {
-            throw new RuntimeException(nsme);
-        } catch (ClassNotFoundException cnfe) {
-            throw new RuntimeException(cnfe);
-        } catch (InstantiationException ie) {
-            throw new RuntimeException(ie);
-        } catch (IllegalAccessException iae) {
-            throw new RuntimeException(iae);
-        } catch (InvocationTargetException ite) {
-            throw new RuntimeException(ite.getCause());
+        } catch (NoSuchMethodException | ClassNotFoundException | InstantiationException
+                | IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
         }
-
-    }    
+    }
 }
diff --git a/core/src/main/java/org/unigram/likelike/lsh/LSHRecommendations.java b/core/src/main/java/org/unigram/likelike/lsh/LSHRecommendations.java
@@ -16,18 +16,11 @@
  */
 package org.unigram.likelike.lsh;
 
-import java.io.BufferedWriter;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.util.Random;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
-
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -36,24 +29,18 @@
 import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.unigram.likelike.common.*;
 
-import org.unigram.likelike.common.Candidate;
-import org.unigram.likelike.common.FsUtil;
-import org.unigram.likelike.common.LikelikeConstants;
-import org.unigram.likelike.common.LikelikeLogger;
-import org.unigram.likelike.common.RelatedUsersWritable;
-import org.unigram.likelike.common.SeedClusterId;
-import org.unigram.likelike.lsh.GetRecommendationsMapper;
-import org.unigram.likelike.lsh.GetRecommendationsReducer;
-import org.unigram.likelike.lsh.SelectClustersMapper;
-import org.unigram.likelike.lsh.SelectClustersReducer;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.Random;
 
 /**
  * Extract recommendations for input examples. 
  */
-public class LSHRecommendations extends
-    Configured implements Tool {
-
+public class LSHRecommendations extends Configured implements Tool {
     /** logger. */
     protected final LikelikeLogger logger 
         = LikelikeLogger.getLogger();
@@ -122,20 +109,20 @@ public int run(final String[] args, final Configuration conf)
                 conf.setLong(LikelikeConstants.MAX_OUTPUT_SIZE, 
                         Long.parseLong(args[++i]));
             } else if ("-storage".equals(args[i])) {
-            	String storageType = args[++i];
-            	if (storageType.equals("dfs")) {
-            		conf.set(LikelikeConstants.LIKELIKE_OUTPUT_WRITER, 
-            				LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER);
-            	} else if  (storageType.equals("cassandra")) {
-            		conf.set(LikelikeConstants.LIKELIKE_OUTPUT_WRITER, 
-            				this.cassandraAccessor);            		
-            	}  
+                String storageType = args[++i];
+                if (storageType.equals("dfs")) {
+                    conf.set(LikelikeConstants.LIKELIKE_OUTPUT_WRITER,
+                            LikelikeConstants.DEFAULT_LIKELIKE_OUTPUT_WRITER);
+                } else if  (storageType.equals("cassandra")) {
+                    conf.set(LikelikeConstants.LIKELIKE_OUTPUT_WRITER,
+                            this.cassandraAccessor);
+                }
             } else if ("-help".equals(args[i])) {
                 this.showParameters();
                 return 0;
             }
         }
-        
+
         this.setHashKeys(iterate, inputFile, conf);
         this.extractClusters(inputFile, clusterDir, conf);
         this.getRecommendations(clusterDir, 
@@ -351,7 +338,5 @@ public static void main(final String[] args)
         int exitCode = ToolRunner.run(
                 new LSHRecommendations(), args);
         System.exit(exitCode);
-    }    
-
-
+    }
 }