1. add on_delete file_to_upload remove file from the upload list. 2. added filename prefix and suffic rules

Author: andrewluiqut

README.md

@@ -89,14 +89,24 @@ uploader.mode: web   # web or headless
 Determines whether the bagfile capturer provides a web interface (`web`) or no interface (`headless`). Other values will cause an error.
 
 ```yaml
-# filestore path
-uploader.filestore.local: /home/qcr/Insync/Bagfiles
+# the local filestore path where new or modified files are uploaded
+uploader.filestore.local: /home/qcr/Bagfiles
+# the target folder on the google drive
 uploader.filestore.remote: /Bagfiles
 ```
 The `uploader.filestore.local` specifies the local directory where new files are to be detected and uploaded. Note that the uploader is only interested in _new_ files. However, touching the files can force the uploader to consider them as new files.
 
 The `uploader.filestore.remote` specifies the target folder to receive the uploaded file on Google Drive.
 
+```yaml
+# exclude files with suffixes and prefixes
+uploader.ignore.suffix:
+  - .active
+uploader.ignore.prefix:
+  - '~'
+```
+The two yaml keys, `uploader.ignore.suffix` and `uploader.ignore.prefix`, specify a list of suffixes and prefixes of filenames that will be ignored. To define more than one suffix or prefix, add more lines under the key as a list. 
+
 ```yaml
 uploader.web.host: 0.0.0.0
 uploader.web.port: 8070

config/uploader_config.yaml

@@ -2,17 +2,22 @@
 ---
 # Operation Mode
 uploader.mode: web # web or headless
-# filestore path
-uploader.filestore.local: /home/qcr/Insync/Bagfiles
+# the local filestore path where new or modified files are uploaded
+uploader.filestore.local: /home/qcr/Bagfiles
+# the target folder on the google drive
 uploader.filestore.remote: /Bagfiles
+# exclude files with suffixes and prefixes
+uploader.ignore.suffix:
+  - .active
+uploader.ignore.prefix:
+  - '~'
 # Upload delay
 uploader.delay: 30 # seconds
 uploader.error_count.max: 5 
 # Dash Agent (uploader) setting
 uploader.web.host: 0.0.0.0
 uploader.web.port: 8070
 uploader.web.launch_browser: False
-uploader.web.auth: True   # Authentication required if web mode
 # Main system timers for the uploader implemented by Dash
 uploader.system.timer: 1  # seconds
 uploader.console.refresh: 5  # seconds

launch/main.launch

@@ -4,7 +4,7 @@
         <!-- Launch the camera monitor node first, which will launch the camera agent -->
         <group if="$(arg run_uploader)"> 
             <param name="mode" type="string" value="$(arg mode)" /> 
-            <node pkg="gdrive_uploader" type="run.py" name="gdrive_uploader_node" output="screen" respawn="true" >
+            <node pkg="gdrive_uploader" type="run.py" name="gdrive_uploader_node" output="screen" respawn="false" >
             </node>
         </group>
 </launch>   

src/uploader/run.py

@@ -35,6 +35,24 @@ def __init__(self, filestore_path):
         self.filestore_path = filestore_path
         self.remote_filestore_path = CONFIG.get('uploader.filestore.remote', '/')
         self.uploader_delay = CONFIG.get('uploader.delay', 30)
+        # setup ignore list
+        self.ignore_suffix = CONFIG.get('uploader.ignore.suffix', None)
+        if self.ignore_suffix is not None and type(self.ignore_suffix) == str:
+            self.ignore_suffix = [self.ignore_suffix]
+        self.ignore_prefix = CONFIG.get('uploader.ignore.prefix', None)
+        if self.ignore_prefix is not None and type(self.ignore_prefix) == str:
+            self.ignore_prefix = [self.ignore_prefix]  
+            
+    def is_in_ignore_lists(self, filename:str) -> bool:
+        if self.ignore_suffix is not None and type(self.ignore_suffix) in (tuple, list):
+            for prefix in self.ignore_suffix:
+                if filename.endswith(prefix):
+                    return True
+        if self.ignore_prefix is not None and type(self.ignore_prefix) in (tuple, list):
+            for prefix in self.ignore_prefix:
+                if filename.startswith(prefix):
+                    return True        
+        return False
     # The callback function when a create event occurs
     def on_created(self, event:FileSystemEvent):
         pass
@@ -44,11 +62,18 @@ def on_modified(self, event:FileSystemEvent):
             # if the event is from a modified file
             local_path = event.src_path
             filename = file_tools.get_filename(local_path)
+            if self.is_in_ignore_lists(filename):
+                return
             parent_path = file_tools.get_parent(local_path)
             sub_path = parent_path[len(self.filestore_path) + 1:]  # the sub_path cannot start with '/' for os.path.join to work
             remote_path = os.path.join(self.remote_filestore_path, sub_path)
             DAO.add_upload_file(local_path, filename, remote_path, int(time.time()) + CONFIG.get('uploader.delay', self.uploader_delay))
-
+    # the callback function when a file is deleted
+    def on_deleted(self, event:FileSystemEvent):
+        if not event.is_directory:
+            # if the event is from a file
+            local_path = event.src_path
+            DAO.remove_upload_file(local_path)
 
 class GDriveUploader(object):
     def __init__(self):

src/uploader/web/dashapp_top.py

@@ -63,7 +63,7 @@ def _define_app(self):
         self._navbar_with_menu = dbc.NavbarSimple(
                     children=[
                         dbc.NavItem(dbc.NavLink('Console', href='/page_console')),  
-                        dbc.NavItem(dbc.NavLink('DB', href='/page_db_browser')),                                              
+                        # dbc.NavItem(dbc.NavLink('DB', href='/page_db_browser')),                                              
                     ],
                     brand=html.Div([html.H3(_APP_NAME), html.H6('Robotics and Autonomous Systems Group, REF, RI, Queensland University of Technology')]),
                     brand_href='/page_console', color='#ffe6cc', className='fs-4 text')
@@ -108,8 +108,8 @@ def display_page(pathname):
             try: 
                 if pathname == '/page_console':
                     page_content = self._console_page.layout()          
-                elif pathname == '/page_db_browser':
-                    page_content = self._db_browse_page.layout()                                        
+                # elif pathname == '/page_db_browser':
+                #     page_content = self._db_browse_page.layout()                                        
                 elif pathname == '/': 
                     page_content = self._console_page.layout()
                 else: # if redirected to unknown link