Kafka Basics Commands

Instructions:

#install zookeper
sudo apt-get install zookeeperd

#check listening port zookeeper
sudo netstat -nlpt | grep ':2181'

telnet 2181

#download kafka specific version
wget http://www.us.apache.org/dist/kafka/0.8.2.2/kafka_2.9.2-0.8.2.2.tgz

#untar for setup
tar xvzf kafka_2.9.2-0.8.2.2.tgz --strip 1

#set kafka bin path in ~/.bashrc

#Start kafka server
nohup /home/kapil/kafka-2.8.0/bin/kafka-server-start.sh /home/kapil/kafka-2.8.0/config/server.properties > /home/kapil/kafka-2.8.0/kafka.log 2>&1 &

netstat -nlpt

#stop zookeper service
sudo service zookeeper stop

#check location for zookeper property
ls /home/kapil/kafka-2.8.0/config

#start zookeeper
/home/kapil/kafka-2.8.0/bin/zookeeper-server-start.sh /home/kapil/kafka-2.8.0/config/zookeeper.properties

#producer
echo "Hello, Kafka kapil here" | /home/kapil/kafka-2.8.0/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic MyTopic > /dev/null

#Consumer
/home/kapil/kafka-2.8.0/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic MyTopic --from-beginning

#topic
/home/kapil/kafka-2.8.0/bin/kafka-topics.sh --list --zookeeper localhost:2181

FILTERS IN HBASE

HBASE FILTER

FILTER TYPE	DESCRIPTION
scan 'emp',{FILTER ="KeyOnlyFilter()"}	This filter does not take any arguments. It returns only the key component of each key-value.
scan 'emp',{FILTER ="FirstKeyOnlyFilter()"}	This filter doesn't take any arguments. It returns only the first key-value from each row.
scan 'emp',{FILTER ="(PrefixFilter ('1'))"}	This filter takes one argument  a prefix of a row key. It returns only those key-values present in a row that starts with the specified row prefix
scan 'emp',{FILTER ="(PrefixFilter ('1')) AND ColumnPrefixFilter('name')"}	This filter takes one argument  a column prefix. It returns only those key-values present in a column that starts with the specified column prefix. The column prefix must be of the form qualifier
scan 'emp',{FILTER ="MultipleColumnPrefixFilter('name','salary')"}	This filter takes a list of column prefixes. It returns key-values that are present in a column that starts with any of the specified column prefixes. Each of the column prefixes must be of the form qualifier
scan 'emp',{FILTER ="ColumnCountGetFilter(1)"}	This filter takes one argument  a limit. It returns the first limit number of columns in the table.
scan 'emp',{FILTER ="PageFilter(1)"}	This filter takes one argument  a page size. It returns page size number of rows from the table.
scan 'emp',{FILTER ="InclusiveStopFilter('1')"}	This filter takes one argument  a row key on which to stop scanning. It returns all key-values present in rows up to and including the specified row.
scan 'emp',{FILTER ="QualifierFilter(=,'binary:name')"}	This filter takes a compare operator and a comparator. It compares each qualifier name with the comparator using the compare operator and if the comparison returns true, it returns all the key-values in that column.
scan 'emp',{FILTER ="MultipleColumnPrefixFilter('name') AND (ValueFilter(=,'binary:kapil'))" }	This filter takes a compare operator and a comparator. It compares each value with the comparator using the compare operator and if the comparison returns true, it returns that key-value.

Hadoop Setup & Installation

# How to Setup virtual box for hadoop
http://www.wikihow.com/Install-Ubuntu-on-VirtualBox

## It will install java source in your machine at /usr/lib/jvm/java-8-oracle
sudo apt-get install oracle-java8-installer

# add user in group
sudo addgroup hadoop
sudo adduser --ingroup hadoop hduser
                           or
# Creat user for hadoop
groupadd hadoop
useradd -G hadoop hduser
passwrd hduser

# Installing SSH
sudo apt-get install openssh-server
or
yum -y install openssh-server openssh-clients


# For updated vi editor
apt-get install vim

# Configuring SSH
# First login with hduser (and from now use only hduser account for further steps)
sudo su hduser

# Generate ssh key for hduser account
ssh-keygen -t rsa -P ""

## Copy id_rsa.pub to authorized keys from hduser
cat $HOME/.ssh/id_rsa.pub >> $HOME/.ssh/authorized_keys
ssh localhost

# Example ssh reference

https://help.ubuntu.com/lts/serverguide/openssh-server.html

# Create folder kapil
Mkdir kapil 

# Move to kapil
cd kapil

# wget haddop tar from below directory
http://apache.claz.org/hadoop/common/.gz

# Download in folder hadoop and untar the tar
tar -xzvf

# Add permision to user
sudo chown hduser -R /kapil/hadoop

## Create Hadoop temp directories for Namenode and Datanode
sudo mkdir -p /kapil/hadoop_tmp/hdfs/namenode
sudo mkdir -p /kapil/hadoop_tmp/hdfs/datanode

## Again assign ownership of this Hadoop temp folder to Hadoop user
sudo chown hduser -R /kapil/hadoop_tmp/

## User profile : Update $HOME/.bashrc
vi ~/.bashrc

## Update hduser configuration file by appending the
## following environment variables at the end of this file.
# -- HADOOP ENVIRONMENT VARIABLES START -- #
export JAVA_HOME=/usr/lib/jvm/jdk1.7.0_51
export HADOOP_HOME=/kapil/hadoop
export PATH=$PATH:$HADOOP_HOME/bin
export PATH=$PATH:$HADOOP_HOME/sbin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
# -- HADOOP ENVIRONMENT VARIABLES END -- #

## Edit Configuration file : vi /kapil/hadoop/etc/hadoop/hadoop-env.sh
## Change below section
# The java implementation to use.
export JAVA_HOME=/usr/lib/jvm/jdk1.7.0_51


## Edit Configuration file :

## Format Namenode by executing below command
hdfs namenode -format

## Start all Hadoop daemons
# Start hdfs daemons
start-dfs.sh

# Start MapReduce daemons:
start-yarn.sh

# Instead both of these above command you can also use
start-all.sh

## Verify Hadoop daemons through JVM command:
jps

## Hadoop administrators.
Open your default browser and visit to the following links.
# For ResourceManager
Http://localhost:8088

# For NameNode
Http://localhost:50070

# See the job history on server
mr-jobhistory-daemon.sh start historyserver

# Execute below command for run default example1
cd /usr/local/hadoop
hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar pi 2 5

# Execute below command for run default example2
 hadoop fs -copyFromLocal /home/hduser/input.txt input
 hadoop fs -ls input
 hadoop fs -cat input/input.txt
 hadoop fs -cat output
 bin\yarn jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar wordcount input output

# Other useful command
export JAVA_HOME=/usr/lib/jvm/jdk1.7.0_51
export PATH=${JAVA_HOME}/bin:${PATH}
export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
export HADOOP_HOME=/kapil/hadoop
export PATH=$PATH:$HADOOP_HOME/bin

#Check list or user
cut -d: -f1 /etc/passwd
#Check list or group
cut -d: -f1 /etc/group

#remove user
userdel userName

sudo usermod -a -G sudo hduser

Hadoop/HDFS utility commands

# List of file
hadoop fs -ls
hadoop fs -ls lsr

# Disk  space of file
hadoop fs -du

# Print Summary of disk usage
hadoop fs -dus

# Move file or dir. location
hadoop fs -mv

# Copy file or dir.
hadoop fs -cp

# Removes file or dir.
hadoop fs -rm

# Removes file or dir. recursively
hadoop fs -rmr

# Copy file from local dir to hdfs dir.
hadoop fs -put
hadoop fs -copyFromLocal

# Copy file or dir. to hdfs dir.
hadoop fs -moveFromLocal

# Copy file from hdfs to local dir.
hadoop fs -get [-crc]
hadoop fs -moveToLocal
hadoop fs -copyToLocal

# Merge & copy file from hdfs to local dir.
hadoop fs -getmerge

# Display the file
hadoop fs -cat

# Display the Content of file
hadoop fs  -text

# Create dir.
hadoop fs -mkdir

# Sets target replication factor for files
hadoop fs -setrep [-R] [-w] rep

# Create file
hadoop fs -touchz

# Returns 1 if path exists otherwise 0.
hadoop fs -test -[ezd]

# Print Formate path like  fileblocksize (%b),filename (%n),block size (%o),replication (%r), and modification date (%y, %Y).
hadoop fs -stat [format]

# Show last 1kb fle stdout.
hadoop fs -tail [-f]

# Set permission on file & dir.
hadoop fs -chmod [-R] mode

# Change the owner permission
hadoop fs -chown [-R] [owner][:[group]]

# Set the owning group
hadoop fs -chgrp [-R] group

# Provide information about commaond.
hadoop fs -help

LinkedHashMapDemo Java program

import java.util.*;
import java.util.Map.Entry;

public class LinkedHashMapDemo {

public static void main (String args[]){

LinkedHashMap lhm = new LinkedHashMap();

lhm.put("kapil", new Double(100.5) );
lhm.put("gaurav", new Double(200.6));
lhm.put("any", new Double(10.5));

Set> ss = lhm.entrySet();
Iterator> i =ss.iterator();

while(i.hasNext()){
Entry m = (Entry) i.next();
System.out.println("Here is key >> "+ m.getKey() );
System.out.println("Here is value >>"+ m.getValue());

}
System.out.println();
     
      double balance = ((Double)lhm.get("kapil")).doubleValue();
      lhm.put("kapil", new Double(balance + 1000));
      System.out.println("kapil new balance: " +
      lhm.get("kapil"));

}

}

Java program for Convert CSV into excel file

import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;

import org.apache.log4j.Logger;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

public class CsvToXsl {
   
    private static DataInputStream myInput;
    private static XSSFWorkbook hwb;
    private static Logger log = Logger.getLogger(
    ResultGenerator.class.getName());

    @SuppressWarnings("deprecation")
public static void readCsvToXsl(String fullPath) throws IOException{
        File f = new File(fullPath +".csv");
        File f1 = new File(fullPath +".xlsx");
        if(f.exists() && !f1.exists()){
        try {
        log.info("Inside method readCsvToXsl for Generation");
        {
           
            ArrayList> arList=null;
            ArrayList al=null;
            String thisLine;
             FileInputStream fis = new FileInputStream(fullPath +".csv");
            myInput = new DataInputStream(fis);
            arList = new ArrayList>();
            while ((thisLine = myInput.readLine()) != null)
            {
             al = new ArrayList();
             String strar[] = thisLine.split(",");
             for(int j=0;j             {
             al.add(strar[j]);
             }
             arList.add(al);
            }

            try
            {
            hwb = new XSSFWorkbook ();
             XSSFSheet sheet = hwb.createSheet("new sheet");
              for(int k=0;k              {
               ArrayList ardata = (ArrayList)arList.get(k);
               Row row = sheet.createRow((short) 0+k);
               for(int p=0;p               {
                Cell cell = row.createCell((short) p);
                String data = ardata.get(p).toString();
                if(data.startsWith("=")){
                 cell.setCellType(Cell.CELL_TYPE_STRING);
                 data=data.replaceAll("\"", "");
                 data=data.replaceAll("=", "");
                 cell.setCellValue(data);
                }else if(data.startsWith("\"")){
                    data=data.replaceAll("\"", "");
                    cell.setCellType(Cell.CELL_TYPE_STRING);
                    cell.setCellValue(data);
                }else{
                    data=data.replaceAll("\"", "");
                    cell.setCellType(Cell.CELL_TYPE_NUMERIC);
                    cell.setCellValue(data);
                }
               }
              }
             FileOutputStream fileOut = new FileOutputStream(fullPath+".xlsx");
             hwb.write(fileOut);
             fileOut.close();
             log.info("Your excel file has been generated");
            } catch ( Exception ex ) {
                 ex.printStackTrace();
            }
            }
        }
       
        catch (IOException e) {
            e.printStackTrace();
            log.info("unbale to find csv file at path");
        }
      }
    }

}

Java program which show directory and file size

import java.io.File;
import java.util.Iterator;
import java.util.Vector;
import javax.swing.JFrame;
import javax.swing.JScrollPane;
import javax.swing.JSplitPane;
import javax.swing.JTextArea;
import javax.swing.JTree;
import javax.swing.event.TreeModelEvent;
import javax.swing.event.TreeModelListener;
import javax.swing.event.TreeSelectionEvent;
import javax.swing.event.TreeSelectionListener;
import javax.swing.tree.TreeModel;
import javax.swing.tree.TreePath;

public class FileTreeFrame extends JFrame {

  /**
*
*/
private static final long serialVersionUID = 1L;
  private JTree fileTree;
  private FileSystemModel fileSystemModel;
  private JTextArea fileDetailsTextArea = new JTextArea();

  public FileTreeFrame(String directory) {
    super("JTree FileSystem Viewer - Impetus.com");
    fileDetailsTextArea.setEditable(false);
    fileSystemModel = new FileSystemModel(new File(directory));
    fileTree = new JTree(fileSystemModel);
    fileTree.setEditable(true);
    fileTree.addTreeSelectionListener(new TreeSelectionListener() {
      public void valueChanged(TreeSelectionEvent event) {
        File file = (File) fileTree.getLastSelectedPathComponent();
        fileDetailsTextArea.setText(getFileDetails(file));
      }
    });
    JSplitPane splitPane = new JSplitPane(JSplitPane.HORIZONTAL_SPLIT, true, new JScrollPane(
        fileTree), new JScrollPane(fileDetailsTextArea));
    getContentPane().add(splitPane);
    setDefaultCloseOperation(EXIT_ON_CLOSE);
    setSize(640, 480);
    setVisible(true);
  }

  private String getFileDetails(File file) {
    if (file == null)
      return "";
    StringBuffer buffer = new StringBuffer();
    buffer.append("Name: " + file.getName() + "\n");
    buffer.append("Path: " + file.getPath() + "\n");
    buffer.append("Size: " + file.length() + "\n");
    return buffer.toString();
  }

  public static void main(String args[]) {
    new FileTreeFrame("d:\\");
  }
}

class FileSystemModel implements TreeModel {
  private File root;

  private Vector listeners = new Vector();

  public FileSystemModel(File rootDirectory) {
    root = rootDirectory;
  }

  public Object getRoot() {
    return root;
  }

  public Object getChild(Object parent, int index) {
    File directory = (File) parent;
    String[] children = directory.list();
    return new TreeFile(directory, children[index]);
  }

  public int getChildCount(Object parent) {
    File file = (File) parent;
    if (file.isDirectory()) {
      String[] fileList = file.list();
      if (fileList != null)
        return file.list().length;
    }
    return 0;
  }

  public boolean isLeaf(Object node) {
    File file = (File) node;
    return file.isFile();
  }

  public int getIndexOfChild(Object parent, Object child) {
    File directory = (File) parent;
    File file = (File) child;
    String[] children = directory.list();
    for (int i = 0; i < children.length; i++) {
      if (file.getName().equals(children[i])) {
        return i;
      }
    }
    return -1;

  }

  public void valueForPathChanged(TreePath path, Object value) {
    File oldFile = (File) path.getLastPathComponent();
    String fileParentPath = oldFile.getParent();
    String newFileName = (String) value;
    File targetFile = new File(fileParentPath, newFileName);
    oldFile.renameTo(targetFile);
    File parent = new File(fileParentPath);
    int[] changedChildrenIndices = { getIndexOfChild(parent, targetFile) };
    Object[] changedChildren = { targetFile };
    fireTreeNodesChanged(path.getParentPath(), changedChildrenIndices, changedChildren);

  }

  private void fireTreeNodesChanged(TreePath parentPath, int[] indices, Object[] children) {
    TreeModelEvent event = new TreeModelEvent(this, parentPath, indices, children);
    Iterator iterator = listeners.iterator();
    TreeModelListener listener = null;
    while (iterator.hasNext()) {
      listener = (TreeModelListener) iterator.next();
      listener.treeNodesChanged(event);
    }
  }

  public void addTreeModelListener(TreeModelListener listener) {
    listeners.add(listener);
  }

  public void removeTreeModelListener(TreeModelListener listener) {
    listeners.remove(listener);
  }

  private class TreeFile extends File {
    /**
*
*/
private static final long serialVersionUID = 1L;

public TreeFile(File parent, String child) {
      super(parent, child);
    }

    public String toString() {
      return getName();
    }
  }
}