consolidator

1

2
import json
3
import math
4
import os
5
import paramiko
6
import datetime
7

8
import requests
9

10
class GuidDescr(object):
11
    def __init__(self,name:str,atime:float,mtime:float,size:int):
12
        self.name = name
13
        self.mtime = mtime
14
        self.atime = atime
15
        self.size = size
16
    @property
17
    def mtime_dt(self):
18
        return datetime.datetime.fromtimestamp(self.mtime)
19
    
20
    def guid_is_changed(self,input_guid)-> tuple[bool,bool]:
21
        """Сравнение двух справочников по времени последней модификации
22

23
        Args:
24
            input_guid (GuidDescr): Входной справочник для сравнения
25

26
        Returns:
27
            tuple[bool,bool]: результат сравнения (Удаленный изменен, Локальный изменен)
28
        """
29
        try:
30
            ing:GuidDescr = input_guid
31
            delta = ing.mtime-self.mtime
32
            # print(f"{ing.name}: {delta} : r {self.size}: l {ing.size}")
33
            # if self.size!=ing.size or delta>3 : 
34
            #     print(f"!!! {self.size!=ing.size} : {delta>3}")
35
            #     return True
36
            # else: return False
37
            if delta>3: return (False,True)
38
            elif delta<0: return (True,False)
39
            else: return (False,False)
40
        except: raise    
41

42
class GuidsSync(object):
43
    """ Класс для синхронизации справочников с сетевым хранилищем """
44
    def __init__(self,host:str,user:str,password:str):
45
        """Конструктор класс (инициализация соединения по ssh)
46

47
        Args:
48
            host (str): хост
49
            user (str): пользователь
50
            password (str): пароль
51
        """
52

53
        try:
54
            self.rhost=host
55
            self.ruser=user
56
            self.rpass=password
57
            self.ssh:paramiko.SSHClient = paramiko.SSHClient()
58
            self.ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
59
            self.ssh.connect(host,username=user,password=password)
60
            self.sftp:paramiko.SFTPClient = self.ssh.open_sftp()
61
            self.local_changed:dict[str,GuidDescr] = {}
62
            self.remote_changed:dict[str,GuidDescr] = {}            
63
            self.remote_dir = None
64
            self.local_dir = None
65
        except: raise
66
    
67
    def set_remote_dir(self,dir:str):
68
        """ Установить директорию справочников на удаленном сервере """
69
        self.remote_dir=dir
70
    
71
    def set_local_dir(self,dir:str):
72
        """ Установить локальную директорию справочников """
73
        self.local_dir=dir
74
            
75
        
76
    def close_connection(self):
77
        """ Закрытие соединения """
78
        try:
79
            self.sftp.close()
80
            self.ssh.close()
81
        except: raise
82
    
83
    def get_remote_files(self,dir:str)->list[GuidDescr]:
84
        try:
85
            lst=[]
86
            for f in self.sftp.listdir(dir):
87
                itemName=f"{dir}/{f}"
88
                stat = self.sftp.stat(itemName)
89
                lst.append(GuidDescr(f,stat.st_atime,stat.st_mtime,stat.st_size))
90
            return lst 
91
        except: raise
92
    
93
    def get_local_files(self,dir:str)->list[GuidDescr]:
94
        try:
95
            lst=[]
96
            for dirname, subdirs, files in os.walk(dir):
97
                for f in files:
98
                        itemName=os.path.join(dirname, f)
99
                        stat =os.stat(itemName)
100
                        lst.append(GuidDescr(f,stat.st_atime,stat.st_mtime,stat.st_size))
101
            return lst
102
        except: raise
103
    
104
    def check(self)->tuple[bool,bool]:
105
        """Сравнение справочников на удаленном сервере с локальными
106

107
        Raises:
108
            Exception: _description_
109

110
        Returns:
111
            tuple[bool,bool]: (Есть изменения на сервере, Есть изменения локально)
112
        """
113
        try:
114
            if self.remote_dir is None or self.local_dir is None: 
115
                raise Exception("Не указаны пути к удаленным или локальным справочника")
116
            result = self.compare_files(self.remote_dir,self.local_dir)
117
            return (bool(result[0]),bool(result[1]))
118
        except: raise
119
        
120
    def compare_files(self,remote_dir:str,local_dir:str)->tuple[dict[str,GuidDescr],dict[str,GuidDescr]]:
121
        try:
122
            rlist = self.get_remote_files(remote_dir)
123
            llist = self.get_local_files(local_dir)
124
            rdict = {}
125
            ldict = {}
126
            for g in rlist: rdict[g.name]=g
127
            for g in llist: ldict[g.name]=g
128
            for key in ldict.keys():
129
               rguid:GuidDescr = rdict.get(key,None)
130
               if rguid is not None:
131
                   changed = rguid.guid_is_changed(ldict[key])
132
                #    print(f"{key} [{changed}]")
133
                   if changed[0]: self.remote_changed[f"{remote_dir}/{key}"] = rdict[key]                   
134
                   if changed[1]: self.local_changed[f"{local_dir}\\{key}"] = ldict[key]
135
            self.remote_dir=remote_dir
136
            self.local_dir=local_dir
137
            return (self.remote_changed,self.local_changed)
138
        except: raise
139
    
140
    def send_to_remote(self):
141
        try:
142
            if self.remote_dir is not None:
143
                for local_file in self.local_changed.keys():
144
                    times = (self.local_changed[local_file].atime,self.local_changed[local_file].mtime)
145
                    remote_file = f"{self.remote_dir}/{self.local_changed[local_file].name}"
146
                    self.sftp.put(local_file,remote_file)
147
                    self.sftp.utime(remote_file,times)
148
                    print(f"{self.local_changed[local_file].name} [sended]")
149
        except: raise
150
        
151
    def get_from_remote(self):
152
        try:
153
            if self.local_dir is not None:
154
                for remote_file in self.remote_changed.keys():
155
                    local_file = f"{self.local_dir}\\{self.remote_changed[remote_file].name}"
156
                    self.sftp.get(remote_file,local_file)
157
                    print(f"{self.remote_changed[remote_file].name} [accepted]")                    
158
        except: raise
159
        
160
    def reconnect(self):
161
        try:
162
            self.sftp.close()
163
            self.ssh.close()
164
            self.ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
165
            self.ssh.connect(hostname=self.rhost,username=self.ruser,password=self.rpass)
166
            self.sftp = self.ssh.open_sftp()
167
            self.local_changed:dict[str,GuidDescr] = {}
168
            self.remote_changed:dict[str,GuidDescr] = {}
169
        except: raise
170
        
171
class ServerList(object):
172
    """  Список файлов на сервере (nginx + json)  """
173
    def __init__(self,data:bytes) -> None:
174
        try:
175
            self.bytes_data = data
176
            tmp_str = self.bytes_data.decode("utf-8")
177
            self.json_data = json.loads(tmp_str)
178
            self.files = {}
179
            for item in self.json_data:
180
                dt = datetime.datetime.strptime(item["mtime"],"%a, %d %b %Y %H:%M:%S %Z")+datetime.timedelta(hours=3)
181
                self.files[item["name"]]=(dt.timestamp(),item["size"])
182
            self.is_empty = not bool(self.files)
183
        except: raise
184
        
185
class LocalList(object):
186
    """ Локальный список файлов """
187
    def __init__(self,dir:str,filter:str = ".xlsx") -> None:
188
        try:
189
            self.dir_list = [fi for fi in os.listdir(dir) if "~$" not in fi and filter in fi]            
190
            self.files={}
191
            for file in self.dir_list:
192
                fullName= f"{dir}\\{file}"
193
                stat = os.stat(fullName)
194
                self.files[file]=(stat.st_mtime,stat.st_size)
195
            self.is_empty = not bool(self.files)
196
        except: raise
197
        
198
class Synchronizer(object):
199
    """ Синхронизатор файлов """
200
    def __init__(self,repo_url:str,local_dir:str,file_filter:str=".xlsx") -> None:
201
        """_summary_
202

203
        Args:
204
            repo_url (str): URL удаленного репозитория
205
            local_dir (str): Локальная папка
206
            file_filter (str, optional): Фильтр локальных файлов. Defaults to ".xlsx".
207
        """
208
        try:
209
            self.repo = repo_url
210
            self.local_dir = local_dir
211
            self.server_files_empty = True
212
            try:
213
                r = requests.get(self.repo)
214
                self.server_files = ServerList(r.content)
215
                self.server_files_empty = self.server_files.is_empty
216
            except: raise Exception("Bad repo")
217
            self.local_files  = LocalList(self.local_dir,filter=file_filter)
218
            self.local_files_empty = self.local_files.is_empty
219
            self.newer_server = {}
220
            self.newer_local  = {}
221
            self.newer_server_exist = False
222
            self.newer_local_exist = False
223
            if not self.local_files_empty and not self.server_files_empty:
224
                for key in self.local_files.files.keys():
225
                    if (self.local_files.files[key][0]-self.server_files.files[key][0]) > 1:
226
                        self.newer_local[key]=self.local_files.files[key]
227
                        self.newer_local_exist = True
228
                    elif (self.local_files.files[key][0]-self.server_files.files[key][0]) < -1: 
229
                        self.newer_server[key]=self.server_files.files[key]
230
                        self.newer_server_exist = True
231
        except: raise
232
        
233
    def sync_local(self):
234
        """ Синхронизировать локальные файлы на сервер """
235
        try:
236
            if self.newer_local_exist:
237
                for key in self.newer_local.keys():
238
                    fullName=f"{self.local_dir}\\{key}"
239
                    stat  = os.stat(fullName)
240
                    with open(fullName,"rb") as datafile:
241
                        r = requests.put(f"{self.repo}{key}",data=datafile)
242
                        dt= math.trunc(datetime.datetime.now().timestamp())
243
                        datafile.close()
244
                        os.utime(fullName,(stat.st_atime,dt))
245
        except: raise
246
        
247
    def sync_server(self):
248
        """ Синхронизировать серверные файлы на локальную машину """
249
        try:
250
            if self.newer_server_exist:
251
                for key in self.newer_server.keys():
252
                    fullName=f"{self.local_dir}\\{key}"
253
                    stat  = os.stat(fullName)
254
                    url = f"{self.repo}{key}"
255
                    r = requests.get(url)
256
                    open(fullName,"wb").write(r.content)
257
                    os.utime(fullName,(stat.st_atime,self.newer_server[key][0]))
258
        except: raise
259
        
260
    def sync_all(self):
261
        """ Синхронизировать все файлы """
262
        try:
263
            self.sync_local()
264
            self.sync_server()
265
        except: raise
266
        
267
    def init_local(self):
268
        """ Загрузить файлы с удаленного хоста в локальную папку """
269
        try:
270
            for key in self.server_files.files.keys():
271
                fullName=f"{self.local_dir}\\{key}"
272
                url = f"{self.repo}{key}"
273
                r = requests.get(url)
274
                open(fullName,"wb").write(r.content)
275
                os.utime(fullName,(self.server_files.files[key][0],self.server_files.files[key][0]))
276
                print(f"'{key}' downloaded {self.server_files.files[key][1]} bytes")               
277
        except: raise
278
        
279
    def init_server(self):
280
        """ Загрузить файлы на удаленный хост из локального хранилища """
281
        try:
282
            for key in self.local_files.files.keys():
283
                fullName=f"{self.local_dir}\\{key}"
284
                stat  = os.stat(fullName)
285
                url = f"{self.repo}{key}"
286
                with open(fullName,"rb") as datafile:
287
                        r = requests.put(url,data=datafile)
288
                        dt= math.trunc(datetime.datetime.now().timestamp())
289
                        datafile.close()
290
                        os.utime(fullName,(stat.st_atime,dt))
291
                print(f"'{key}' uploaded {self.local_files.files[key][1]} bytes")
292
        except: raise    
293
        
294