Python_Assistant/hadoop_module.py at main · muskan399/Python_Assistant · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import os
from greeting_module import close


def hadoop():
    print("------------Welcome to APACHE HADOOP SERVICES------------")
    print("----You are the Client----")
    while True:
        os.system("clear")
        print("""\n
        1. Setup Hadoop Cluster
        2. Get details of all slaves
        3. List all files in hadoop cluster
        4. Upload a file
        5. Read a file
        6. Remove a file
        7. Back to main menu
        8. Exit
        """)
        inp = int(input("Enter your choice: "))
        if inp == 1:

            #Configuring Master
            ip_master = input("Enter ip of master ")
            os.system("ssh-copy-id -i /root/.ssh/id_rsa.pub {}".format(ip_master))
            os.system("scp jdk-8u171-linux-x64.rpm {}:/".format(ip_master))
            os.system("scp hadoop-1.2.1-1.x86_64.rpm {}:/".format(ip_master))
            os.system("ssh {} rpm -i /jdk-8u171-linux-x64.rpm".format(ip_master))
            os.system("ssh {} rpm -i /hadoop-1.2.1-1.x86_64.rpm --force".format(ip_master))
            os.system("ssh {} mkdir /nn".format(ip_master))
            os.system("ssh {} echo -e \"`sed -i '$d' /etc/hadoop/hdfs-site.xml`<property>\n<name>dfs.name.dir</name>\n<value>/nn</value>\n</property>\n</configuration>\" >> /etc/hadoop/hdfs-site.xml".format(ip_master))
            os.system("ssh {} echo -e \"`sed -i '$d' /etc/hadoop/core-site.xml`<property>\n<name>fs.default.name</name>\n<value>hdfs://{}:9001</value>\n</property>\n</configuration>\" >> /etc/hadoop/core-site.xml".format(ip_master,ip_master))
            os.system("ssh {} hadoop namenode -format".format(ip_master))
            os.system("ssh {} hadoop-daemon start namenode".format(ip_master))
            os.system("ssh {} systemctl stop firewalld".format(ip_master))

            #Configuring Slaves
            no_of_slaves = int(input("Enter how many slaves you want ?"))
            for i in range(1,no_of_slaves+1):
                ip_slave = input("Enter ip of slave ")
                os.system("ssh-copy-id -i /root/.ssh/id_rsa.pub {}".format(ip_slave))
                os.system("scp jdk-8u171-linux-x64.rpm {}:/".format(ip_slave))
                os.system("scp hadoop-1.2.1-1.x86_64.rpm {}:/".format(ip_slave))
                os.system("ssh {} rpm -i /jdk-8u171-linux-x64.rpm".format(ip_slave))
                os.system("ssh {} rpm -i /hadoop-1.2.1-1.x86_64.rpm --force".format(ip_slave))
                os.system("ssh {} mkdir /dn{}".format(ip_slave,i))
                os.system("ssh {} echo -e \"`sed -i '$d' /etc/hadoop/hdfs-site.xml`<property>\n<name>dfs.data.dir</name>\n<value>/dn{}</value>\n</property>\n</configuration>\" >> /etc/hadoop/hdfs-site.xml".format(ip_slave,i))
                os.system("ssh {} echo -e \"`sed -i '$d' /etc/hadoop/core-site.xml`<property>\n<name>fs.default.name</name>\n<value>hdfs://{}:9001</value>\n</property>\n</configuration>\" >> /etc/hadoop/core-site.xml".format(ip_slave,ip_master))
                os.system("ssh {} hadoop-daemon start datanode".format(ip_slave))

            #Configuring Client
            os.system("rpm -i jdk-8u171-linux-x64.rpm")
            os.system("rpm -i hadoop-1.2.1-1.x86_64.rpm")
            os.system("echo -e \"`sed -i '$d' /etc/hadoop/core-site.xml`<property>\n<name>fs.default.name</name>\n<value>hdfs://{}:9001</value>\n</property>\n</configuration>\" >> /etc/hadoop/core-site.xml".format(ip_master))

        elif inp == 2:
            os.system("hadoop dfsadmin -report")

        elif inp == 3:
            os.system("hadoop fs -ls  /")

        elif inp == 4:
            filename = input("Enter filename")
            os.system("hadoop fs -put {} /".format(filename))

        elif inp == 5:
            filename = input("Enter filename")
            os.system("hadoop fs -cat {} /".format(filename))

        elif inp == 6:
            filename = input("Enter filename")
            os.system("hadoop fs -rm {} /".format(filename))
        elif inp==7:
            return
        elif inp== 8:
            close()
        os.system("clear")