From f7a8962ecab540913fb153dbaf6dfd443350d9aa Mon Sep 17 00:00:00 2001 From: Derek Weitzel Date: Tue, 9 Jun 2026 12:25:03 -0500 Subject: [PATCH] Switch all remote access from XRootD to WebDAV Replace the XRootD.client library with webdav4 (httpx-based) for every remote operation: directory listing, file reading, and metadata. All target repositories are globally read-only over HTTPS, so no authentication is required, and the same host/port serve WebDAV. bin/cvmfs_sync: - Drop XRootD.client; add webdav4 + xml.etree. - propfind_dir(): one Depth:1 PROPFIND per directory, parsed by XML local-name, returning name / is_dir / size / executable (Apache mod_dav executable property; falls back to 0644). - process_download_file(): stream the file via an httpx GET into the existing cvmfs_swissknife graft pipe, preserving deadline checks, EPIPE handling, child cleanup, and chmod. - Remove lookup_dataserver(); HTTP redirects (redirector -> data server) are followed transparently. - Remove the unused GridFTP/uberftp checksum path. - Keep the multiprocessing download pool and threaded listing workers. config/*.config: rewrite source URLs from root:// to https:// (same host/port). cvmfs-sync-driver: drop gridftp_source handling. cvmfs-sync.spec: Requires python3-webdav4 instead of xrootd-python; update the user description text. Co-Authored-By: Claude Opus 4.8 --- bin/cvmfs_sync | 377 ++++++++++-------- config/cvmfs-sync.spec | 4 +- config/des.osgstorage.org.config | 4 +- config/dune.osgstorage.org.config | 6 +- config/et-gw.osgstorage.org.config | 2 +- config/gluex.osgstorage.org.config | 4 +- config/gwosc.osgstorage.org.config | 2 +- config/icarus.osgstorage.org.config | 4 +- config/icecube.osgstorage.org.config | 2 +- config/jlab.osgstorage.org.config | 4 +- config/kagra.storage.igwn.org.config | 2 +- config/ligo-test.storage.igwn.org.config | 2 +- config/ligo.storage.igwn.org.config | 2 +- config/minerva.osgstorage.org.config | 6 +- config/mu2e.osgstorage.org.config | 6 +- config/nova.osgstorage.org.config | 8 +- config/public-uc.osgstorage.org.config | 2 +- config/sbn.osgstorage.org.config | 4 +- config/sbnd.osgstorage.org.config | 4 +- config/scitokens.osgstorage.org.config | 2 +- ...sdsc-nrp-osdf-origin.osgstorage.org.config | 2 +- config/shared.storage.igwn.org.config | 2 +- config/snomass21.osgstorage.org.config | 8 +- config/uboone.osgstorage.org.config | 6 +- config/uhawaii.osgstorage.org.config | 4 +- config/virgo.storage.igwn.org.config | 2 +- update-scripts/cvmfs-sync-driver | 8 +- 27 files changed, 254 insertions(+), 225 deletions(-) diff --git a/bin/cvmfs_sync b/bin/cvmfs_sync index 0515b20..9869236 100755 --- a/bin/cvmfs_sync +++ b/bin/cvmfs_sync @@ -15,20 +15,10 @@ import fnmatch import optparse import urllib.parse import threading -import subprocess import multiprocessing +import xml.etree.ElementTree as ET -try: - import subprocess32 -except ImportError: - subprocess32 = None - -oldpath = sys.path -sys.path = sys.path[1:] - -import XRootD.client - -sys.path = oldpath +from webdav4.client import Client g_failed_files = [] g_processed_files = [] @@ -179,15 +169,131 @@ def should_skip(input_url, output_filename, size, output_mode): return False -def process_download_file(xrootd_url, output_filename, output_mode, deadline): +# Per-operation socket timeout (seconds) for the underlying httpx client. +WEBDAV_TIMEOUT = 300 + +# PROPFIND body requesting just the properties we need: whether the entry is a +# collection (directory), its size, and the Apache mod_dav "executable" flag. +PROPFIND_BODY = ( + '' + '' + '' + '' +) + +# Cache of webdav4 clients keyed by connection (scheme://netloc). This is a +# per-process cache; download worker processes each build their own client so +# that httpx connections are never shared across forks. +_g_client_cache = {} + + +def make_client(connection): + """Create a webdav4 client for a connection (e.g. https://host:port). + + Redirects are followed so that storage redirectors transparently hand off + to the actual data server. """ - Download an entire file to the local host and stream its contents through cvmfs_swissknife - in order to create checksums. + return Client(connection, follow_redirects=True, timeout=WEBDAV_TIMEOUT) + + +def get_client(connection): + """Return a cached webdav4 client for the connection, creating one if needed.""" + client = _g_client_cache.get(connection) + if client is None: + client = make_client(connection) + _g_client_cache[connection] = client + return client + + +def _localname(tag): + """Strip the XML namespace from an ElementTree tag.""" + return tag.split("}", 1)[1] if "}" in tag else tag + + +class WebDAVEntry(object): + """A single child entry returned by a directory listing.""" + __slots__ = ("name", "is_dir", "size", "executable") + + def __init__(self, name, is_dir, size, executable): + self.name = name + self.is_dir = is_dir + self.size = size + self.executable = executable + + +def propfind_dir(client, path): + """ + List a single directory level over WebDAV via a PROPFIND with Depth: 1. + + Returns a list of WebDAVEntry, one per child (the directory itself is + excluded). Raises an exception (whose message contains "permission denied" + for 401/403) when the listing fails. + """ + url = client.join_url(path) + resp = client.http.request("PROPFIND", url, content=PROPFIND_BODY, + headers={"Depth": "1", "Content-Type": "application/xml"}, + follow_redirects=True) + if resp.status_code in (401, 403): + raise Exception("Permission denied (HTTP %d) when listing %s" % (resp.status_code, path)) + resp.raise_for_status() + + root = ET.fromstring(resp.content) + req_path = urllib.parse.urlsplit(str(url)).path.rstrip("/") + entries = [] + for response in root: + if _localname(response.tag) != "response": + continue + href = None + size = 0 + is_dir = False + executable = False + have_props = False + for child in response: + ln = _localname(child.tag) + if ln == "href": + href = child.text + elif ln == "propstat": + status = "".join(s.text or "" for s in child if _localname(s.tag) == "status") + if "200" not in status: + continue + have_props = True + for prop in child: + if _localname(prop.tag) != "prop": + continue + for el in prop: + eln = _localname(el.tag) + if eln == "resourcetype": + if any(_localname(g.tag) == "collection" for g in el): + is_dir = True + elif eln == "getcontentlength" and el.text: + try: + size = int(el.text) + except ValueError: + pass + elif eln == "executable" and el.text: + executable = el.text.strip().upper() == "T" + if not have_props or not href: + continue + # Resolve the href (which may be percent-encoded and/or an absolute URL) + # down to a server path, then take its basename. + href_path = urllib.parse.unquote(urllib.parse.urlsplit(href).path).rstrip("/") + if href_path == req_path: + continue # the directory itself + name = href_path.rsplit("/", 1)[-1] + entries.append(WebDAVEntry(name, is_dir, size, executable)) + return entries + + +def process_download_file(connection, server_path, output_filename, output_mode, deadline): """ + Download an entire file over WebDAV and stream its contents through + cvmfs_swissknife in order to create checksums. + """ + url = connection + server_path if (deadline > 0) and (time.time() > deadline): - raise Exception("URL %s timed out when still in processing queue." % xrootd_url) + raise Exception("URL %s timed out when still in processing queue." % url) - logging.info("Grafting %s to %s" % (xrootd_url, output_filename)) + logging.info("Grafting %s to %s" % (url, output_filename)) # Fork cvmfs_swissknife readfp, writefp = os.pipe() @@ -199,115 +305,77 @@ def process_download_file(xrootd_url, output_filename, output_mode, deadline): args = ['cvmfs_swissknife', 'graft', '-i', '-', '-o', output_filename, '-Z', 'none', '-c', '24'] os.execvp("cvmfs_swissknife", args) finally: - raise Exception("Failed to exec cvmfs_swissknife for processing URL %s" % xrootd_url) - - # Open remote file - # After fork to reduce risk of xrootd thread cleanup deadlock - with XRootD.client.File() as fp: - status = fp.open(xrootd_url)[0] - if status.status: - # Cleanup cvmfs_swissknife - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - raise Exception("Failed to open file (%s) in Xrootd: %s" % (xrootd_url, status)) + raise Exception("Failed to exec cvmfs_swissknife for processing URL %s" % url) - # Check deadline again; file-open may have taken awhile for an unresponsive server. - if (deadline > 0) and (time.time() > deadline): - # Cleanup cvmfs_swissknife - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - raise Exception("URL %s timed out while being opened." % xrootd_url) - - os.close(readfp) - next_offset = 0 - while True: - if (deadline > 0) and (time.time() > deadline): - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - raise Exception("Timed out when transferring URL %s" % xrootd_url) - - status, response = fp.read(offset=next_offset, size=1024*1024) - if status.status: - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - raise Exception("Failed to create read request for %s; status: %s; killing graft %d" % (xrootd_url, str(status), pid)) - - # Empty response indicates EOF. - if (response == None) or (len(response) == 0): - logging.debug("Finished processing URL %s" % xrootd_url) - os.close(writefp) - os.waitpid(pid, 0) - os.chmod(output_filename, output_mode) - break - else: + os.close(readfp) + client = get_client(connection) + + # Check deadline again; the queue may have backed up. + if (deadline > 0) and (time.time() > deadline): + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + raise Exception("URL %s timed out before the download started." % url) + + next_offset = 0 + try: + with client.http.stream("GET", client.join_url(server_path), follow_redirects=True) as resp: + if resp.status_code in (401, 403): + raise Exception("Permission denied (HTTP %d) for URL %s" % (resp.status_code, url)) + resp.raise_for_status() + for chunk in resp.iter_bytes(chunk_size=1024*1024): + if (deadline > 0) and (time.time() > deadline): + raise Exception("Timed out when transferring URL %s" % url) + if not chunk: + continue try: - os.write(writefp, response) + os.write(writefp, chunk) except OSError as oe: if oe.errno != errno.EPIPE: raise logging.error("cvmfs_swissknife process unexpectedly died.") + os.close(writefp) + os.waitpid(pid, 0) return False - next_offset += len(response) - return next_offset + next_offset += len(chunk) + except Exception: + # Clean up the graft child before propagating the failure. + try: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except OSError: + pass + try: + os.close(writefp) + except OSError: + pass + raise + logging.debug("Finished processing URL %s" % url) + os.close(writefp) + os.waitpid(pid, 0) + os.chmod(output_filename, output_mode) + return next_offset -def process_checksum_file(gridftp_url, xrootd_url, output_filename, output_mode, deadline): - """ - Process and create a CVMFS checksum / graft file. - If a GridFTP server is available, use that to generate a checksum file. Otherwise, - utilize Xrootd to download the file to memory and checksum it locally. +def process_checksum_file(connection, server_path, output_filename, output_mode, deadline): """ - if not gridftp_url: - return process_download_file(xrootd_url, output_filename, output_mode, deadline) - - if (deadline > 0) and (time.time() > deadline): - raise Exception("URL %s timed out when still in processing queue." % xrootd_url) - - parsed = urllib.parse.urlparse(gridftp_url) - logging.info("Querying GridFTP server for %s" % gridftp_url) - try: - if subprocess32: - output = subprocess32.check_output(["uberftp", parsed.netloc, "quote cksm cvmfs 0 -1 %s" % parsed.path], timeout=5*60) - else: - output = subprocess.check_output(["uberftp", parsed.netloc, "quote cksm cvmfs 0 -1 %s" % parsed.path]) - except subprocess.CalledProcessError as cpe: # https://bugs.python.org/issue9400 - CalledProcessError messes up multiprocessing - raise Exception("UberFTP call failed. (cmd=%s, output=%s, returncode=%s)" % (cpe.cmd, cpe.output, str(cpe.returncode))) - except Exception as e: - logging.error("Unhandled exception %s" % str(e)) - raise Exception("UberFTP call failed with an unhandled exception.") - cvmfs_cksum = None - for line in output.splitlines(): - line = line.strip() - if line.startswith("500"): - raise Exception(line) - elif line.startswith("213 "): - cvmfs_cksum = line[4:] - if cvmfs_cksum: - with open(output_filename, "w") as fp: - pass - graftfile = cvmfs_cksum.replace(";", "\n") - with open(graft_filename(output_filename), "w") as fp: - fp.write(graftfile + "\n") - else: - raise Exception("Remote GridFTP server did not return a CVMFS checksum") - return 0 + Process and create a CVMFS checksum / graft file by downloading the file + over WebDAV and streaming it through cvmfs_swissknife. + """ + return process_download_file(connection, server_path, output_filename, output_mode, deadline) -def process_files(base_url, gridftp_base_url, output_dir, count, ignore=[], include=[], thread_count=2, deadline=0): +def process_files(base_url, output_dir, count, ignore=[], include=[], thread_count=2, deadline=0): global g_bytes_xfer - current = 0 - worklist = [] - - xrootd_base_parsed = urllib.parse.urlparse(base_url) - xrootd_connection_info = xrootd_base_parsed.scheme + "://" + xrootd_base_parsed.netloc - xrootd_directory = xrootd_base_parsed.path + base_parsed = urllib.parse.urlparse(base_url) + connection = base_parsed.scheme + "://" + base_parsed.netloc + directory = base_parsed.path # First, try to retrieve the file pool = multiprocessing.Pool(count) futures = [] - for filename, statinfo in process_dir(xrootd_connection_info, xrootd_directory, output_dir, ignore=ignore, include=include, thread_count=thread_count): + for filename, size, executable in process_dir(connection, directory, output_dir, ignore=ignore, include=include, thread_count=thread_count): if deadline and (time.time() > deadline): logging.error("Passed deadline when scanning directory tree") break @@ -315,16 +383,11 @@ def process_files(base_url, gridftp_base_url, output_dir, count, ignore=[], incl #if len(futures) >= 200: logging.info("Queue of 50k files to checksum has been reached; will attempt remainder next time.") break - if gridftp_base_url: - gridftp_url = gridftp_base_url + filename - else: - gridftp_url = None - xrootd_url = base_url + filename + server_path = directory + filename output_filename = output_dir + filename input_url = base_url + filename - size = statinfo.size - if (statinfo.flags & XRootD.client.flags.StatInfoFlags.X_BIT_SET): + if executable: output_mode = 0o755 # Is executable, set 755 else: output_mode = 0o644 # Else 644 @@ -333,7 +396,7 @@ def process_files(base_url, gridftp_base_url, output_dir, count, ignore=[], incl continue if should_ignore_filename(filename): continue - future = pool.apply_async(process_checksum_file, (gridftp_url, xrootd_url, output_filename, output_mode, deadline)) + future = pool.apply_async(process_checksum_file, (connection, server_path, output_filename, output_mode, deadline)) future.filename = filename future.size = size futures.append(future) @@ -354,37 +417,13 @@ def process_files(base_url, gridftp_base_url, output_dir, count, ignore=[], incl if "permission denied" in str(e).lower(): print("Skipping permission denied error") else: - g_failed_files.append(gridftp_base_url + "/" + future.filename) - #g_failed_files.append(gridftp_base_url + "/" + future.filename) + g_failed_files.append(base_url + "/" + future.filename) futures = timed_out_futures logging.info("All jobs processed - closing up pool.") pool.join() -def lookup_dataserver(redirector, path, timeout=10): - """ - Look up an individual data server from a redirector. - If lookup fails, return the original redirector. - - For sites where the redirector does not have the cluster filesystem - mounted, dirlist requests will fail. Use this function to select - a data server. - """ - - fs = XRootD.client.FileSystem(redirector) - - handler = XRootD.client.utils.AsyncResponseHandler() - status = fs.stat(path, timeout=timeout, callback=handler) - status, _, hostlist = handler.wait() - - if status.ok: - # Return the last URL in the hostlist (the data server) - return(str(list(hostlist)[-1].url)) - else: - return redirector - - -def list_dir(work_queue, results_queue, fs): +def list_dir(work_queue, results_queue, client): """ Helper thread for listing directories in parallel """ @@ -392,33 +431,34 @@ def list_dir(work_queue, results_queue, fs): cwd = work_queue.get() logging.debug("%s working on directory listing of %s" % (threading.current_thread().name, cwd)) starttime = time.time() - status = None - dirlist = None + entries = None + error = None try: - status, dirlist = fs.dirlist(cwd, - flags=XRootD.client.flags.DirListFlags.STAT) + entries = propfind_dir(client, cwd) except Exception as e: - logging.error("Exception encoutered when listing directory %s: %s" % (cwd, str(e))) + error = e + logging.error("Exception encountered when listing directory %s: %s" % (cwd, str(e))) finally: - results_queue.put((status, dirlist, cwd, starttime), True) + results_queue.put((error, entries, cwd, starttime), True) work_queue.task_done() -def start_listing_workers(thread_count, work_queue, results_queue, fs): +def start_listing_workers(thread_count, work_queue, results_queue, client): """ Helper function for launching the directory spider threads """ for count in range(thread_count): - t = threading.Thread(target=list_dir, args=(work_queue, results_queue, fs), name="Directory list worker %d" % count) + t = threading.Thread(target=list_dir, args=(work_queue, results_queue, client), name="Directory list worker %d" % count) t.setDaemon(True) t.start() -def process_dir(base_url, directory, output_dir, ignore=[], include=[], thread_count=2): +def process_dir(connection, directory, output_dir, ignore=[], include=[], thread_count=2): global g_failed_dirs - base_data_url = lookup_dataserver(base_url, directory) - fs = XRootD.client.FileSystem(base_data_url) - logging.info("Switching from redirector %s to data server %s" % (base_url, base_data_url)) + # A single webdav4 client shared across the listing threads; httpx clients + # are safe for concurrent use across threads. HTTP redirects are followed + # transparently, so no explicit redirector-to-dataserver lookup is needed. + client = make_client(connection) logging.info("Starting processing of top-level directory %s", directory) worklist = [directory] base_len = len(directory) @@ -429,7 +469,7 @@ def process_dir(base_url, directory, output_dir, ignore=[], include=[], thread_c work_sent = 0 results_queue = queue.Queue() # shouldn't block results_received = 0 - start_listing_workers(thread_count, work_queue, results_queue, fs) + start_listing_workers(thread_count, work_queue, results_queue, client) while worklist or (work_sent != results_received): @@ -451,20 +491,19 @@ def process_dir(base_url, directory, output_dir, ignore=[], include=[], thread_c work_queue.put(cwd, block=True) # Process results serially. - status, dirlist, cwd, starttime = results_queue.get(True) + error, entries, cwd, starttime = results_queue.get(True) logging.info("Processing entries of %s" % cwd) results_queue.task_done() results_received += 1 - if status.status: - logging.warning("Failed to list directory (%s) skipping: %s" % (cwd, status)) - if "permission denied" in str(status).lower(): + if error is not None or entries is None: + logging.warning("Failed to list directory (%s) skipping: %s" % (cwd, error)) + if error is not None and "permission denied" in str(error).lower(): print("Skipping permission denied error") else: g_failed_dirs.append(str(cwd)) continue - xrootd_cwd_names = set() - entries = list(dirlist.dirlist) + cwd_names = set() logging.debug("Directory %s has %d entries" % (cwd, len(entries))) if cwd == top_level_dir: print(("%s is the top level directory" % (cwd))) @@ -476,13 +515,12 @@ def process_dir(base_url, directory, output_dir, ignore=[], include=[], thread_c for entry in entries: if should_ignore_path(cwd + "/" + entry.name, ignore, include): continue - # Sometimes the remote XRootD will prefix the name with '/' - xrootd_cwd_names.add(entry.name.split("/")[-1]) - if entry.statinfo.flags & XRootD.client.flags.StatInfoFlags.IS_DIR: + cwd_names.add(entry.name) + if entry.is_dir: worklist.append(cwd + "/" + entry.name) else: fname = cwd + "/" + entry.name - yield (fname[base_len:], entry.statinfo) + yield (fname[base_len:], entry.size, entry.executable) cwd_output = output_dir + "/" + cwd[len(directory):] try: cwd_output_names = set(os.listdir(cwd_output)) @@ -492,7 +530,7 @@ def process_dir(base_url, directory, output_dir, ignore=[], include=[], thread_c continue else: raise - remaining_names = cwd_output_names.difference(xrootd_cwd_names) + remaining_names = cwd_output_names.difference(cwd_names) for name in remaining_names: # Ignore presence of hidden files (such as the graft files). if name.startswith("."): continue @@ -529,12 +567,9 @@ def main(): for src in srcs: if time.time() > deadline: break - source_filelist = [] - info = src.split(",") - gridftp_src = '' - if len(info) == 2: - src, gridftp_src = info - process_files(src, gridftp_src, dest, concurrency, ignore=opts.ignore, include=opts.include, thread_count=opts.metadata_concurrency, deadline=deadline) + # Tolerate a legacy "src,gridftp_src" form by ignoring the second field. + src = src.split(",")[0] + process_files(src, dest, concurrency, ignore=opts.ignore, include=opts.include, thread_count=opts.metadata_concurrency, deadline=deadline) processing_time = time.time() - starttime print("Total of %.1f GB in %d files processed in %.2f seconds." % (g_bytes_xfer/(1024.**3), len(g_processed_files), processing_time)) diff --git a/config/cvmfs-sync.spec b/config/cvmfs-sync.spec index 933b70e..76ce249 100644 --- a/config/cvmfs-sync.spec +++ b/config/cvmfs-sync.spec @@ -10,7 +10,7 @@ URL: https://github.com/bbockelm/cvmfs-sync Source0: %{name}-%{version}.tar.gz Requires: cvmfs-server -Requires: xrootd-python +Requires: python3-webdav4 Requires(pre): shadow-utils %{?systemd_requires} @@ -58,7 +58,7 @@ install -d $RPM_BUILD_ROOT/%{_localstatedir}/cache/cvmfs-sync getent group cvmfs-sync >/dev/null || groupadd -r cvmfs-sync getent passwd cvmfs-sync >/dev/null || \ useradd -r -g cvmfs-sync -d /var/lib/cvmfs-sync -s /sbin/nologin \ - -c "User to synchronize CVMFS with a remote XRootD server namespace" cvmfs-sync + -c "User to synchronize CVMFS with a remote WebDAV server namespace" cvmfs-sync exit 0 diff --git a/config/des.osgstorage.org.config b/config/des.osgstorage.org.config index 0308503..6fb5cb6 100644 --- a/config/des.osgstorage.org.config +++ b/config/des.osgstorage.org.config @@ -3,6 +3,6 @@ repo = des.osgstorage.org [Sync DES] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/des/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/des/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/des/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/des/persistent/stash/ destination = /cvmfs/des.osgstorage.org/pnfs/fnal.gov/usr/des/persistent/stash/ diff --git a/config/dune.osgstorage.org.config b/config/dune.osgstorage.org.config index 5324b59..25d766e 100644 --- a/config/dune.osgstorage.org.config +++ b/config/dune.osgstorage.org.config @@ -3,7 +3,7 @@ repo = dune.osgstorage.org [Sync mu2e] -#source = root://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/dune/persistent/stash/ -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/dune/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/dune/persistent/stash/ +#source = https://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/dune/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/dune/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/dune/persistent/stash/ destination = /cvmfs/dune.osgstorage.org/pnfs/fnal.gov/usr/dune/persistent/stash/ diff --git a/config/et-gw.osgstorage.org.config b/config/et-gw.osgstorage.org.config index 15f1f4f..610a2a7 100644 --- a/config/et-gw.osgstorage.org.config +++ b/config/et-gw.osgstorage.org.config @@ -2,5 +2,5 @@ repo = et-gw.osgstorage.org [Sync et-gw MDC1] -source = root://et-origin.cism.ucl.ac.be:1094//et-gw/PUBLIC/MDC1 +source = https://et-origin.cism.ucl.ac.be:1094//et-gw/PUBLIC/MDC1 destination = /cvmfs/et-gw.osgstorage.org/et-gw/PUBLIC/MDC1 diff --git a/config/gluex.osgstorage.org.config b/config/gluex.osgstorage.org.config index ac92905..e1fd7cc 100644 --- a/config/gluex.osgstorage.org.config +++ b/config/gluex.osgstorage.org.config @@ -3,7 +3,7 @@ repo = gluex.osgstorage.org [Sync Gluex-uconn0] -source = root://nod25.phys.uconn.edu:1094//gluex/uconn0 +source = https://nod25.phys.uconn.edu:1094//gluex/uconn0 destination = /cvmfs/gluex.osgstorage.org/gluex/uconn0 include = concurrency = 10 @@ -11,7 +11,7 @@ max_time = 600 ignore = *.tmp, *.swp [Sync Gluex-uconn1] -source = root://cn445.storrs.hpc.uconn.edu:1094//gluex/uconn1 +source = https://cn445.storrs.hpc.uconn.edu:1094//gluex/uconn1 destination = /cvmfs/gluex.osgstorage.org/gluex/uconn1 include = concurrency = 10 diff --git a/config/gwosc.osgstorage.org.config b/config/gwosc.osgstorage.org.config index 82238f1..be5aa34 100644 --- a/config/gwosc.osgstorage.org.config +++ b/config/gwosc.osgstorage.org.config @@ -3,6 +3,6 @@ repo = gwosc.osgstorage.org [Sync gwosc] -source = root://origin.ligo.caltech.edu:1094/ +source = https://origin.ligo.caltech.edu:1094/ destination = /cvmfs/gwosc.osgstorage.org/ concurrency = 50 diff --git a/config/icarus.osgstorage.org.config b/config/icarus.osgstorage.org.config index 62a8c0d..5972278 100644 --- a/config/icarus.osgstorage.org.config +++ b/config/icarus.osgstorage.org.config @@ -3,6 +3,6 @@ repo = icarus.osgstorage.org [Sync icarus] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/icarus/persistent/stash -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/icarus/persistent/stash +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/icarus/persistent/stash +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/icarus/persistent/stash destination = /cvmfs/icarus.osgstorage.org/pnfs/fnal.gov/usr/icarus/persistent/stash diff --git a/config/icecube.osgstorage.org.config b/config/icecube.osgstorage.org.config index 5698f71..ea09081 100644 --- a/config/icecube.osgstorage.org.config +++ b/config/icecube.osgstorage.org.config @@ -3,5 +3,5 @@ repo = icecube.osgstorage.org [Sync icecube] -source = root://stash-origin.icecube.wisc.edu//icecube/PUBLIC +source = https://stash-origin.icecube.wisc.edu//icecube/PUBLIC destination = /cvmfs/icecube.osgstorage.org/icecube/PUBLIC diff --git a/config/jlab.osgstorage.org.config b/config/jlab.osgstorage.org.config index e291806..d5f7d51 100644 --- a/config/jlab.osgstorage.org.config +++ b/config/jlab.osgstorage.org.config @@ -3,7 +3,7 @@ repo = jlab.osgstorage.org [Sync jlab] -source = root://sci-xrootd.jlab.org:2094//jlab +source = https://sci-xrootd.jlab.org:2094//jlab destination = /cvmfs/jlab.osgstorage.org/jlab include = concurrency = 10 @@ -11,7 +11,7 @@ concurrency = 10 max_time = 30 [Sync eic] -source = root://sci-xrootd.jlab.org:2094//eic +source = https://sci-xrootd.jlab.org:2094//eic destination = /cvmfs/jlab.osgstorage.org/eic include = #concurrency = 10 diff --git a/config/kagra.storage.igwn.org.config b/config/kagra.storage.igwn.org.config index 585b187..d181d8a 100644 --- a/config/kagra.storage.igwn.org.config +++ b/config/kagra.storage.igwn.org.config @@ -7,5 +7,5 @@ x509_credential = /tmp/x509up_u1221 command = /usr/libexec/cvmfs-sync/ligo-auth-gen {authz_output} -p ZZZ_PASSWORD_ZZZ -w /usr/share/cvmfs-sync/kagra_authz [Sync kagra] -source = root://kagra-dsr-b2.icrr.u-tokyo.ac.jp:1095//igwn/kagra/ +source = https://kagra-dsr-b2.icrr.u-tokyo.ac.jp:1095//igwn/kagra/ destination = /cvmfs/kagra.storage.igwn.org/igwn/kagra/ diff --git a/config/ligo-test.storage.igwn.org.config b/config/ligo-test.storage.igwn.org.config index e0d3372..193d8f0 100644 --- a/config/ligo-test.storage.igwn.org.config +++ b/config/ligo-test.storage.igwn.org.config @@ -7,6 +7,6 @@ x509_credential = /tmp/x509up_u1221 command = /usr/libexec/cvmfs-sync/ligo-auth-gen {authz_output} -p ZZZ_PASSWORD_ZZZ -w /usr/share/cvmfs-sync/ligo_storage_authz [Sync ligo] -source = root://origin-readtest.ligo.caltech.edu:1095//igwn/test/ligo +source = https://origin-readtest.ligo.caltech.edu:1095//igwn/test/ligo destination = /cvmfs/ligo-test.storage.igwn.org/igwn/test/ligo concurrency = 20 diff --git a/config/ligo.storage.igwn.org.config b/config/ligo.storage.igwn.org.config index cc06032..408d136 100644 --- a/config/ligo.storage.igwn.org.config +++ b/config/ligo.storage.igwn.org.config @@ -7,6 +7,6 @@ x509_credential = /tmp/x509up_u1221 command = /usr/libexec/cvmfs-sync/ligo-auth-gen {authz_output} -p ZZZ_PASSWORD_ZZZ -w /usr/share/cvmfs-sync/ligo_storage_authz [Sync ligo] -source = root://origin-ifo.ligo.caltech.edu:1095//igwn/ligo/ +source = https://origin-ifo.ligo.caltech.edu:1095//igwn/ligo/ destination = /cvmfs/ligo.storage.igwn.org/igwn/ligo/ concurrency = 20 diff --git a/config/minerva.osgstorage.org.config b/config/minerva.osgstorage.org.config index 281a3ed..2fc6830 100644 --- a/config/minerva.osgstorage.org.config +++ b/config/minerva.osgstorage.org.config @@ -3,7 +3,7 @@ repo = minerva.osgstorage.org [Sync mu2e] -#source = root://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/minerva/persistent/stash/ -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/minerva/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/minerva/persistent/stash/ +#source = https://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/minerva/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/minerva/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/minerva/persistent/stash/ destination = /cvmfs/minerva.osgstorage.org/pnfs/fnal.gov/usr/minerva/persistent/stash/ diff --git a/config/mu2e.osgstorage.org.config b/config/mu2e.osgstorage.org.config index 6d454b9..3a82fa1 100644 --- a/config/mu2e.osgstorage.org.config +++ b/config/mu2e.osgstorage.org.config @@ -3,7 +3,7 @@ repo = mu2e.osgstorage.org [Sync mu2e] -#source = root://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/mu2e/persistent/stash/ -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/mu2e/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/mu2e/persistent/stash/ +#source = https://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/mu2e/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/mu2e/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/mu2e/persistent/stash/ destination = /cvmfs/mu2e.osgstorage.org/pnfs/fnal.gov/usr/mu2e/persistent/stash/ diff --git a/config/nova.osgstorage.org.config b/config/nova.osgstorage.org.config index 02ab270..5f5b789 100644 --- a/config/nova.osgstorage.org.config +++ b/config/nova.osgstorage.org.config @@ -3,15 +3,15 @@ repo = nova.osgstorage.org #[Sync flux] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/nova/data/flux/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/nova/data/flux/ #destination = /cvmfs/nova.osgstorage.org/pnfs/fnal.gov/usr/nova/data/flux/ [Sync analysis] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/analysis/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/analysis/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/analysis/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/analysis/ destination = /cvmfs/nova.osgstorage.org/pnfs/fnal.gov/usr/nova/persistent/analysis/ ignore = /pnfs/fnal.gov/usr/nova/persistent/analysis/nux/nus_cmf [Sync stash] -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/nova/persistent/stash/ destination = /cvmfs/nova.osgstorage.org/pnfs/fnal.gov/usr/nova/persistent/stash/ diff --git a/config/public-uc.osgstorage.org.config b/config/public-uc.osgstorage.org.config index 23309db..1937413 100644 --- a/config/public-uc.osgstorage.org.config +++ b/config/public-uc.osgstorage.org.config @@ -3,6 +3,6 @@ repo = public-uc.osgstorage.org [Sync public-uc] -source = root://pelican-osdf-public.tempest.uchicago.edu:8443//ospool/uc-shared/public +source = https://pelican-osdf-public.tempest.uchicago.edu:8443//ospool/uc-shared/public destination = /cvmfs/public-uc.osgstorage.org/ospool/uc-shared/public concurrency = 50 diff --git a/config/sbn.osgstorage.org.config b/config/sbn.osgstorage.org.config index 3c7191d..39275a9 100644 --- a/config/sbn.osgstorage.org.config +++ b/config/sbn.osgstorage.org.config @@ -3,7 +3,7 @@ repo = sbn.osgstorage.org [Sync sbn] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/sbn/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/sbn/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/sbn/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/sbn/persistent/stash/ destination = /cvmfs/sbn.osgstorage.org/pnfs/fnal.gov/usr/sbn/persistent/stash/ diff --git a/config/sbnd.osgstorage.org.config b/config/sbnd.osgstorage.org.config index 393cb98..01b0d5c 100644 --- a/config/sbnd.osgstorage.org.config +++ b/config/sbnd.osgstorage.org.config @@ -3,7 +3,7 @@ repo = sbnd.osgstorage.org [Sync sbnd] -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/sbnd/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/sbnd/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/sbnd/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/sbnd/persistent/stash/ destination = /cvmfs/sbnd.osgstorage.org/pnfs/fnal.gov/usr/sbnd/persistent/stash/ diff --git a/config/scitokens.osgstorage.org.config b/config/scitokens.osgstorage.org.config index cc28647..5cd0a83 100644 --- a/config/scitokens.osgstorage.org.config +++ b/config/scitokens.osgstorage.org.config @@ -12,7 +12,7 @@ file = /etc/cvmfs-sync/scitokens.authz # Skipping, as recommended by Tom Downes #[Sync postO2] -#source = root://xrootd-local.unl.edu//user/ligo/frames/postO2/ +#source = https://xrootd-local.unl.edu//user/ligo/frames/postO2/ #destination = /cvmfs/ligo.osgstorage.org/frames/postO2/ diff --git a/config/sdsc-nrp-osdf-origin.osgstorage.org.config b/config/sdsc-nrp-osdf-origin.osgstorage.org.config index 02ef19a..5e28544 100644 --- a/config/sdsc-nrp-osdf-origin.osgstorage.org.config +++ b/config/sdsc-nrp-osdf-origin.osgstorage.org.config @@ -2,7 +2,7 @@ repo = sdsc-nrp-osdf-origin.osgstorage.org [Sync nrp] -source = root://sdsc-origin.nationalresearchplatform.org:1094//nrp +source = https://sdsc-origin.nationalresearchplatform.org:1094//nrp destination = /cvmfs/sdsc-nrp-osdf-origin.osgstorage.org/nrp concurrency = 10 max_time = 600 diff --git a/config/shared.storage.igwn.org.config b/config/shared.storage.igwn.org.config index 97bf505..8a033d8 100644 --- a/config/shared.storage.igwn.org.config +++ b/config/shared.storage.igwn.org.config @@ -7,6 +7,6 @@ x509_credential = /tmp/x509up_u1221 command = /usr/libexec/cvmfs-sync/ligo-auth-gen {authz_output} -p ZZZ_PASSWORD_ZZZ -w /usr/share/cvmfs-sync/shared_igwn_authz [Sync ligo] -source = root://origin-shared.ligo.caltech.edu:1095//igwn/shared/ +source = https://origin-shared.ligo.caltech.edu:1095//igwn/shared/ destination = /cvmfs/shared.storage.igwn.org/igwn/shared/ #concurrency = 20 diff --git a/config/snomass21.osgstorage.org.config b/config/snomass21.osgstorage.org.config index d24388e..75d0e13 100644 --- a/config/snomass21.osgstorage.org.config +++ b/config/snomass21.osgstorage.org.config @@ -3,7 +3,7 @@ repo = snomass21.osgstorage.org #[Sync Merra2] -#source = root://stashcache-origin-merra2.nautilus.optiputer.net:31094//merra2 +#source = https://stashcache-origin-merra2.nautilus.optiputer.net:31094//merra2 #destination = /cvmfs/stash.osgstorage.org/merra2 #include = #concurrency = 50 @@ -11,7 +11,7 @@ repo = snomass21.osgstorage.org #ignore = *.tmp, *.swp #[Sync Gluex] -#source = root://nod25.phys.uconn.edu//Gluex +#source = https://nod25.phys.uconn.edu//Gluex #destination = /cvmfs/stash.osgstorage.org/Gluex #include = #concurrency = 10 @@ -19,7 +19,7 @@ repo = snomass21.osgstorage.org #ignore = *.tmp, *.swp #[Sync chtc] -#source = root://sc-origin.chtc.wisc.edu//chtc/PUBLIC +#source = https://sc-origin.chtc.wisc.edu//chtc/PUBLIC #destination = /cvmfs/stash.osgstorage.org/chtc/PUBLIC #include = #concurrency = 10 @@ -27,7 +27,7 @@ repo = snomass21.osgstorage.org #ignore = *.tmp, *.swp #[Sync Stash] -#source = root://stash.osgconnect.net//osgconnect/public/ +#source = https://stash.osgconnect.net//osgconnect/public/ #destination = /cvmfs/stash.osgstorage.org/osgconnect/public/ #include = /user/*/public/* #ignore = *.tmp, *.swp, /user/*/public/public, /user/krieger, /user/nepomuk, /user/toth diff --git a/config/uboone.osgstorage.org.config b/config/uboone.osgstorage.org.config index acb5bce..9782692 100644 --- a/config/uboone.osgstorage.org.config +++ b/config/uboone.osgstorage.org.config @@ -3,7 +3,7 @@ repo = uboone.osgstorage.org [Sync uboone] -#source = root://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/uboone/persistent/stash/ -#source = root://stashcache.fnal.gov//pnfs/fnal.gov/usr/uboone/persistent/stash/ -source = root://stashredirector.fnal.gov//pnfs/fnal.gov/usr/uboone/persistent/stash/ +#source = https://fndca1.fnal.gov:1095/pnfs/fnal.gov/usr/uboone/persistent/stash/ +#source = https://stashcache.fnal.gov//pnfs/fnal.gov/usr/uboone/persistent/stash/ +source = https://stashredirector.fnal.gov//pnfs/fnal.gov/usr/uboone/persistent/stash/ destination = /cvmfs/uboone.osgstorage.org/pnfs/fnal.gov/usr/uboone/persistent/stash/ diff --git a/config/uhawaii.osgstorage.org.config b/config/uhawaii.osgstorage.org.config index 9ef6c90..5bc5a2b 100644 --- a/config/uhawaii.osgstorage.org.config +++ b/config/uhawaii.osgstorage.org.config @@ -4,11 +4,11 @@ repo = uhawaii.osgstorage.org [Sync me2022] -source = root://prp01.ifa.hawaii.edu//uhawaii/manoa/ifa/me2022 +source = https://prp01.ifa.hawaii.edu//uhawaii/manoa/ifa/me2022 destination = /cvmfs/uhawaii.osgstorage.org/uhawaii/manoa/ifa/me2022 #concurrency = 10 #ignore = sent2/*, sent1/*, c3/*, tfr/* [Sync hinode] -source = root://prp01.ifa.hawaii.edu//uhawaii/manoa/ifa/hinode +source = https://prp01.ifa.hawaii.edu//uhawaii/manoa/ifa/hinode destination = /cvmfs/uhawaii.osgstorage.org/uhawaii/manoa/ifa/hinode diff --git a/config/virgo.storage.igwn.org.config b/config/virgo.storage.igwn.org.config index eef302f..cd77bc4 100644 --- a/config/virgo.storage.igwn.org.config +++ b/config/virgo.storage.igwn.org.config @@ -7,5 +7,5 @@ x509_credential = /tmp/x509up_u1221 command = /usr/libexec/cvmfs-sync/ligo-auth-gen {authz_output} -p ZZZ_PASSWORD_ZZZ -w /usr/share/cvmfs-sync/virgo_authz [Sync virgo] -source = root://ingrid-se09.cism.ucl.ac.be:1095//igwn/virgo/ +source = https://ingrid-se09.cism.ucl.ac.be:1095//igwn/virgo/ destination = /cvmfs/virgo.storage.igwn.org/igwn/virgo/ diff --git a/update-scripts/cvmfs-sync-driver b/update-scripts/cvmfs-sync-driver index 80c94f0..3644449 100755 --- a/update-scripts/cvmfs-sync-driver +++ b/update-scripts/cvmfs-sync-driver @@ -53,9 +53,6 @@ class CvmfsSyncDriver(object): if config.has_option(section, "include"): self.include = re.split(r",?\s*", config.get(section, "include")) self.source = config.get(section, "source") - self.gridftp_source = None - if config.has_option(section, "gridftp_source"): - self.gridftp_source = config.get(section, "gridftp_source") self.destination = config.get(section, "destination") def run(self): @@ -74,10 +71,7 @@ class CvmfsSyncDriver(object): if self.include: command += ["--include", ",".join(self.include)] command += ["--verbose"] - source = self.source - if self.gridftp_source: - source += "," + self.gridftp_source - command += [source, self.destination] + command += [self.source, self.destination] process = subprocess.Popen(command) process.wait() return process.returncode