recurse folders for mart server

projmon/ftpexplorer.py

@@ -7,6 +7,7 @@ from time import sleep
 from paho.mqtt import client as mqtt_client
 from logging import Logger
 from uuid import uuid5, NAMESPACE_OID
+import copy
 
 class FTPDirInfo(BaseModel):
     name: str
@@ -20,6 +21,7 @@ class FTPServer(BaseModel):
     username: Optional[str] = None
     password: Optional[str] = None
     cwd: Optional[str] = None
+    recurse: Optional[bool] = False
     timeout: int = 15
     dirs: DirDescriptors = Field(default_factory=dict)
     notify_filters: List[str] = Field(default_factory=list)
@@ -65,7 +67,8 @@ def initialize_ftp_servers(file_path: str) -> FTPServers:
         host="192.168.31.15",
         port=21,
         notify_filters=["TLR", "TRL", "FTR"],
-        cwd='TRAILER'
+        cwd='TRAILER',
+        recurse=True
         ))
 
     dcp_servers.servers.append(FTPServer(
@@ -80,6 +83,19 @@ def initialize_ftp_servers(file_path: str) -> FTPServers:
     save_ftp_servers_to_json(dcp_servers, file_path)
     return dcp_servers
 
+def explore_dir(ftp_connection: FTP, dir_descriptors: DirDescriptors):
+    content: List[str] = ftp_connection.nlst()
+    if "ASSETMAP" in content or "ASSETMAP.xml" in content:
+        dir_name: str = ftp_connection.pwd()
+        unique = uuid5(namespace=NAMESPACE_OID,name=dir_name).hex.upper()[:11]
+        dir_descriptors[unique] = FTPDirInfo(name=ftp_connection.pwd())
+    else:
+        for d in content:
+            ftp_connection.cwd(d)
+            explore_dir(ftp_connection=ftp_connection, dir_descriptors=dir_descriptors)
+            ftp_connection.cwd("..")
+        
+
 def list_new_directories(ftp_server: FTPServer) -> DirDescriptors:
     """Lists new directories on the FTP server that are not already known."""
     with ftp_server.connect() as ftp_connection:
@@ -94,19 +110,29 @@ def list_new_directories(ftp_server: FTPServer) -> DirDescriptors:
                 modified_time_str: str = dir_info.get('modify', '')
                 modified_time: datetime = datetime.strptime(modified_time_str, '%Y%m%d%H%M%S') if modified_time_str else datetime.now()
                 unique: str = dir_info.get('unique', str(datetime.now().timestamp()))
-                new_dirs[unique]=FTPDirInfo(name=dir_name, 
-                                           modified_time=modified_time)
+                new_dirs[unique] = FTPDirInfo(name=dir_name, modified_time=modified_time)
         except error_perm:
             # revert to simple directory listing mode
             for d in ftp_connection.nlst():
                 dir_name = d
-                unique = uuid5(namespace=NAMESPACE_OID,name=dir_name).hex.upper()[:11]
-                new_dirs[unique]=FTPDirInfo(name=dir_name)
+                unique = uuid5(namespace=NAMESPACE_OID, name=dir_name).hex.upper()[:11]
+                new_dirs[unique] = FTPDirInfo(name=dir_name)
                                 
-        
         diff_dirs: DirDescriptors = { k:v for k,v in new_dirs.items() if k not in ftp_server.dirs }
+        
+        # recurse only in difference directory so it is faster
+        if ftp_server.recurse:
+            base_dir: str = ftp_connection.pwd()
+            recurse_dirs: DirDescriptors = copy.deepcopy(diff_dirs)
+            for u, d in recurse_dirs.items():
+                ftp_connection.cwd(base_dir)
+                ftp_connection.cwd(d.name)
+                explore_dir(ftp_connection=ftp_connection, dir_descriptors=diff_dirs)
+                del diff_dirs[u]
+            new_dirs |= diff_dirs
+        
         ftp_server.dirs = new_dirs  # Update the server's directory list with the new directories
-        return diff_dirs  # Return both new directories and the difference
+        return diff_dirs  # Return the difference
                          
 def check_new_titles(mqtt: mqtt_client.Client, topic: str, file_path: str, logger: Logger) -> None:
     dcp_servers = FTPServers()
@@ -140,4 +166,20 @@ def check_new_titles(mqtt: mqtt_client.Client, topic: str, file_path: str, logge
     # Save server information with updated DCP list
     save_ftp_servers_to_json(dcp_servers, file_path)
     return
+
+if __name__ == "__main__":
+    dcp_servers = FTPServers()
+    first_run = False
+    file_path: str = "./dcpservers.json"
+    if os.path.exists(file_path):
+        dcp_servers = load_ftp_servers_from_json(file_path)
+    else:
+        dcp_servers = initialize_ftp_servers(file_path)
+        first_run = True
+        
+    for server in dcp_servers.servers:
+        with server.connect() as ftp_connection:
+            this_dirs: DirDescriptors = {}
+            explore_dir(ftp_connection, this_dirs)
+            print(this_dirs)