Merge pull request #6 from yanghongfei/master

[Update]做一些简单的优化

biz/handlers/admin_user_handler.py

@@ -7,7 +7,6 @@
 
 
 import json
-import tornado.web
 from libs.base_handler import BaseHandler
 from models.server import AdminUser, model_to_dict
 from websdk.db_context import DBContext

biz/handlers/asset_server_handler.py

@@ -3,7 +3,7 @@
 # @Time    : 2019/4/17 16:09
 # @Author  : Fred Yangxiaofei
 # @File    : asset_server_handler.py
-# @Role    : 说明脚本功能
+# @Role    : 主机管理
 
 
 import json
@@ -15,7 +15,6 @@
 from tornado import gen
 from concurrent.futures import ThreadPoolExecutor
 from tornado.concurrent import run_on_executor
-# from libs.server_test import RsyncPublicKey, start_rsync, get_server_sysinfo
 from libs.common import check_ip
 from libs.server.sync_to_tagtree import main as sync_tag_tree
 import datetime
@@ -385,8 +384,6 @@ def sync_task(self):
 
     @gen.coroutine
     def get(self, *args, **kwargs):
-        # msg = yield self.sync_task()
-
         try:
             # 超过120s 返回Timeout
             msg = yield gen.with_timeout(datetime.timedelta(seconds=120), self.sync_task(),

biz/handlers/asset_tag_handler.py

@@ -7,7 +7,6 @@
 
 
 import json
-import tornado.web
 from libs.base_handler import BaseHandler
 from models.server import Tag, Server, ServerTag, model_to_dict
 from models.db import DB, DBTag

biz/handlers/hand_update_asset_handler.py

@@ -26,31 +26,30 @@ class HandUpdateAssetHandler(BaseHandler):
 
     @run_on_executor(executor='_thread_pool')
     def asset_update(self, id_list):
-        # time.sleep(20)  #测试下阻塞
-        server_list = []
         # 检查下状态，是true的话直接推送资产
         with DBContext('r', None, True) as session:
             for i in id_list:
 
-                server_info = session.query(Server.ip, Server.port, AdminUser.system_user,
+                server_list = session.query(Server.ip, Server.port, AdminUser.system_user,
                                             AdminUser.user_key, Server.state).outerjoin(AdminUser,
                                                                                         AdminUser.admin_user == Server.admin_user).filter(
                     Server.id == i).all()
-                server_list.append(server_info)
-                # server_info = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')]
-                server_state = server_info[0][4]
+                # server_list = [('47.100.231.147', 22, 'root', '-----BEGIN RSA PRIVATE KEYxxxxxEND RSA PRIVATE KEY-----', 'false')]
+                server_state = server_list[0][4]
                 if server_state == 'true':
-                    ins_log.read_log('info', 'update asset info')
+                    ins_log.read_log('info', '机器状态为True，开始更新资产')
                     # 选中的是true，直接更新资产
                     asset_data = get_server_sysinfo(server_list)
+                    ins_log.read_log('info', '资产信息：{}'.format(asset_data))
                     update_asset(asset_data)
 
                 else:
                     # 选中的是其余的状态，那就先推送key, 拿到推送成功的key，更新资产
-                    ins_log.read_log('info', 'rsync public key , update asset info')
+                    ins_log.read_log('info', '状态为非True，先推送公钥再进行更新资产')
                     rsync_sucess_list = rsync_public_key(server_list)
                     if rsync_sucess_list:
                         asset_data = get_server_sysinfo(server_list)
+                        ins_log.read_log('info', '资产信息：{}'.format(asset_data))
                         update_asset(asset_data)
 
     @gen.coroutine
@@ -60,11 +59,10 @@ def post(self, *args, **kwargs):
         if not id_list:
             return self.write(dict(code=1, msg='关键参数不能为空'))
 
-        #yield self.asset_update(id_list)
         try:
-            #超过120s 返回Timeout
+            # 超过120s 返回Timeout
             yield gen.with_timeout(datetime.timedelta(seconds=120), [self.asset_update(id_list)],
-                                                  quiet_exceptions=tornado.gen.TimeoutError)
+                                   quiet_exceptions=tornado.gen.TimeoutError)
         except gen.TimeoutError:
             return self.write(dict(code=-2, msg='TimeOut'))
         return self.write(dict(code=0, msg='任务执行完成，提醒： 完成状态为：True, 错误状态：False， False状态下可点击查看日志进行排错'))

biz/handlers/system_user_handler.py

@@ -7,12 +7,10 @@
 
 
 import json
-import tornado.web
 from libs.base_handler import BaseHandler
 from models.server import SystemUser, model_to_dict
 from websdk.db_context import DBContext
 from opssdk.operate import MyCryptV2
-# from libs.test import MyCrypt
 from libs.common import exec_shell, is_number
 from libs.server.push_system_user import PushSystemUser
 import shortuuid

libs/aliyun/ecs.py

@@ -3,7 +3,7 @@
 # @Time    : 2019/5/13 14:02
 # @Author  : Fred Yangxiaofei
 # @File    : ecs.py
-# @Role    :  获取Aliyun资产信息推送到CMDB
+# @Role    : 获取Aliyun资产信息推送到CMDB
 
 
 import json
@@ -113,7 +113,7 @@ def get_server_info(self):
             asset_data['os_name'] = i.get('OSName')
             server_list.append(asset_data)
             # print(asset_data)
-            ins_log.read_log('info', asset_data)
+            ins_log.read_log('info', '资产信息:{}'.format(asset_data))
         return server_list
 
     def sync_cmdb(self):

libs/aws/ec2.py

@@ -51,7 +51,6 @@ def get_server_info(self):
         if ret:
             for r in ret:
                 for i in r['Instances']:
-                    print(i)
                     asset_data = dict()
                     try:
                         asset_data['hostname'] = i.get('Tags')[0].get('Value')
@@ -81,7 +80,7 @@ def sync_cmdb(self):
             return False
         with DBContext('w') as session:
             for server in server_list:
-                print(server)
+                ins_log.read_log('info', '资产信息:{}'.format(server))
                 ip = server.get('public_ip')
                 instance_id = server.get('instance_id', 'Null')
                 hostname = server.get('hostname', instance_id)

libs/common.py

@@ -9,7 +9,6 @@
 import time
 import subprocess
 import paramiko
-import shortuuid
 import concurrent.futures
 import re
 from  libs.ansibleAPI.runner import Runner

libs/qcloud/cvm.py

@@ -125,7 +125,7 @@ def get_server_info(self):
             asset_data['os_type'] = os_type
             # print(asset_data)
             server_list.append(asset_data)
-            ins_log.read_log('info', asset_data)
+            ins_log.read_log('info', '资产信息:{}'.format(asset_data))
         return server_list
 
     def sync_cmdb(self):

libs/server/asset_auto_update.py

@@ -8,7 +8,6 @@
 
 from libs.db_context import DBContext
 from models.server import Server, ServerDetail, model_to_dict, AdminUser, AssetErrorLog
-# from libs.server_test import start_rsync, get_server_sysinfo, RsyncPublicKey
 from libs.server.sync_public_key import RsyncPublicKey, start_rsync
 from libs.server.collect_asset_info import get_server_sysinfo
 from libs.web_logs import ins_log
@@ -42,7 +41,7 @@ def rsync_public_key(self):
         id_list = self.check_server_state()
         if not id_list:
             ins_log.read_log('info', '[PASS]: No new server found, automatically skipping push public key')
-          #  print('[PASS]: No new server found, automatically skipping push public key')
+            #  print('[PASS]: No new server found, automatically skipping push public key')
 
             return
 
@@ -90,83 +89,81 @@ def rsync_public_key(self):
 
         return rsync_sucess_list
 
-    def get_host_info(self):
-        '''获取主机信息'''
-        id_list = self.check_server_state()
-        connect_server_list = []
-        with DBContext('r') as session:
-            for i in id_list:
-                connect_info = session.query(Server.ip, Server.port, AdminUser.system_user).outerjoin(AdminUser,
-                                                                                                      AdminUser.admin_user == Server.admin_user).filter(
-                    Server.id == i).all()
-                connect_server_list.append(connect_info)
-        res_data_list = get_server_sysinfo(connect_server_list)
-        return res_data_list
-
     def update_asset(self, host_info):
         """
         更新资产到数据库
         :param host_data: 主机返回的资产采集基础数据
         :return:
         """
         with DBContext('w') as session:
-            for host in host_info:
-                for k, v in host.items():
-                    try:
-                        if host[k].get('status'):
-                            _sn = v.get('sn', None)
-                            _hostname = v.get('host_name', None)
-                            _cpu = v.get('cpu', None)
-                            _cpu_cores = v.get('cpu_cores', None)
-                            _memory = v.get('memory', None)
-                            _disk = v.get('disk', None)
-                            _os_type = v.get('os_type', None)
-                            _os_kernel = v.get('os_kernel', None)
-                            # _instance_id = v.get('instance_id', None)
-                            # _instance_type = v.get('instance_type', None)
-                            # _instance_state = v.get('instance_state', None)
-
-                            exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first()
-                            if not exist_detail:
-                                # 不存在就新建
-                                new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores,
-                                                                 memory=_memory, disk=_disk,
-                                                                 os_type=_os_type, os_kernel=_os_kernel)
-                                session.add(new_server_detail)
-                                session.commit()
-                                session.query(Server).filter(Server.ip == k).update(
-                                    {Server.hostname: _hostname, Server.state: 'true'})
-                                session.commit()
-                            else:
-                                # 存在就更新
-                                session.query(ServerDetail).filter(ServerDetail.ip == k).update({
-                                    ServerDetail.sn: _sn, ServerDetail.ip: k,
-                                    ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores,
-                                    ServerDetail.disk: _disk, ServerDetail.memory: _memory,
-                                    ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel,
-                                })
-
-                                session.query(Server).filter(Server.ip == k).update(
-                                    {Server.hostname: _hostname, Server.state: 'true'})
-                                session.commit()
-                    except sqlalchemy.exc.IntegrityError as e:
-                        ins_log.read_log('error',e)
-                        # 状态改为Flse->删除主机Detail--记录错误信息
-                        session.query(Server).filter(Server.ip == k).update({Server.state: 'false'})
-                        session.query(ServerDetail).filter(ServerDetail.ip == k).delete(
-                            synchronize_session=False)
-
-                        exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first()
-                        error_log = str(e)
-                        if exist_ip:
-                            session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update(
-                                {AssetErrorLog.error_log: error_log})
+            for k, v in host_info.items():
+                try:
+                    if host_info[k].get('status'):
+                        _sn = v.get('sn', None)
+                        _hostname = v.get('host_name', None)
+                        _cpu = v.get('cpu', None)
+                        _cpu_cores = v.get('cpu_cores', None)
+                        _memory = v.get('memory', None)
+                        _disk = v.get('disk', None)
+                        _os_type = v.get('os_type', None)
+                        _os_kernel = v.get('os_kernel', None)
+                        # _instance_id = v.get('instance_id', None)
+                        # _instance_type = v.get('instance_type', None)
+                        # _instance_state = v.get('instance_state', None)
+
+                        exist_detail = session.query(ServerDetail).filter(ServerDetail.ip == k).first()
+                        if not exist_detail:
+                            # 不存在就新建
+                            new_server_detail = ServerDetail(ip=k, sn=_sn, cpu=_cpu, cpu_cores=_cpu_cores,
+                                                             memory=_memory, disk=_disk,
+                                                             os_type=_os_type, os_kernel=_os_kernel)
+                            session.add(new_server_detail)
+                            session.commit()
+                            session.query(Server).filter(Server.ip == k).update(
+                                {Server.hostname: _hostname, Server.state: 'true'})
+                            session.commit()
                         else:
-                            new_error_log = AssetErrorLog(ip=k, error_log=error_log)
-                            session.add(new_error_log)
+                            # 存在就更新
+                            session.query(ServerDetail).filter(ServerDetail.ip == k).update({
+                                ServerDetail.sn: _sn, ServerDetail.ip: k,
+                                ServerDetail.cpu: _cpu, ServerDetail.cpu_cores: _cpu_cores,
+                                ServerDetail.disk: _disk, ServerDetail.memory: _memory,
+                                ServerDetail.os_type: _os_type, ServerDetail.os_kernel: _os_kernel,
+                            })
+
+                            session.query(Server).filter(Server.ip == k).update(
+                                {Server.hostname: _hostname, Server.state: 'true'})
+                            session.commit()
+                except sqlalchemy.exc.IntegrityError as e:
+                    ins_log.read_log('error', e)
+                    # 状态改为Flse->删除主机Detail--记录错误信息
+                    session.query(Server).filter(Server.ip == k).update({Server.state: 'false'})
+                    session.query(ServerDetail).filter(ServerDetail.ip == k).delete(
+                        synchronize_session=False)
+
+                    exist_ip = session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).first()
+                    error_log = str(e)
+                    if exist_ip:
+                        session.query(AssetErrorLog).filter(AssetErrorLog.ip == k).update(
+                            {AssetErrorLog.error_log: error_log})
+                    else:
+                        new_error_log = AssetErrorLog(ip=k, error_log=error_log)
+                        session.add(new_error_log)
 
-                        session.commit()
-                        return False
+                    session.commit()
+                    return False
+
+    def get_host_info(self):
+        '''获取主机信息，并写入数据库'''
+        id_list = self.check_server_state()
+        with DBContext('r') as session:
+            for i in id_list:
+                server_list = session.query(Server.ip, Server.port, AdminUser.system_user).outerjoin(AdminUser,
+                                                                                                     AdminUser.admin_user == Server.admin_user).filter(
+                    Server.id == i).all()
+                asset_data = get_server_sysinfo(server_list)
+                ins_log.read_log('info', '资产信息：{}'.format(asset_data))
+                self.update_asset(asset_data)
 
 
 def main(state):
@@ -192,14 +189,9 @@ def main(state):
         if not obj.rsync_public_key():
             # 如果没有发现有新增的主机,直接PASS
             return
-        hosts_data = obj.get_host_info()
-        obj.update_asset(hosts_data)
+        obj.get_host_info()
     elif state == 'true':
-        hosts_data_list = obj.get_host_info()
-        obj.update_asset(hosts_data_list)
-
-    else:
-        pass
+        obj.get_host_info()
 
 
 def new_tail_data():
@@ -211,5 +203,4 @@ def true_tail_data():
 
 
 if __name__ == '__main__':
-    main('new')
-# fire.Fire(main)
+    fire.Fire(main)

libs/server/collect_asset_info.py

@@ -25,7 +25,9 @@ def get_host_info(server_list):
             module_args="",
             remote_user=user,
             pattern="all",
-            hosts=ip
+            hosts=ip,
+            timeout=10,
+            forks=10
         )
 
         result = runner.run()
@@ -116,18 +118,22 @@ def get_host_info(server_list):
             asset_data[ip]['os_type'] = os_type
             asset_data[ip]['os_kernel'] = os_kernel
 
-        print(asset_data)
+        # print(asset_data)
+        # print('ok')
         return asset_data
 
 
 def get_server_sysinfo(server_list):
-    """
-    多进程采集机器信息
-    :param server_list: 主机列表
-    :return:
-    """
-    #print(list(exec_thread(func=get_host_info, iterable1=server_list)))
-    return list(exec_thread(func=get_host_info, iterable1=server_list))
+    return get_host_info(server_list)
+
+# def get_server_sysinfo(server_list):
+#     """
+#     多进程采集机器信息
+#     :param server_list: 主机列表
+#     :return:
+#     """
+#     #print(list(exec_thread(func=get_host_info, iterable1=server_list)))
+#     return list(exec_thread(func=get_host_info, iterable1=server_list))
 
 
 if __name__ == '__main__':