CCDB: add support for bearer token (#15711)

Allows authenticating with the soon to be deployed security proxy of the CI.

Also make the endpoint configurable in tests.

Solves the issues with CCDB on SLC10.

Author: ktf

CCDB/src/CcdbApi.cxx

@@ -60,6 +60,29 @@ using namespace std;
 std::mutex gIOMutex; // to protect TMemFile IO operations
 unique_ptr<TJAlienCredentials> CcdbApi::mJAlienCredentials = nullptr;
 
+namespace
+{
+/// Append the gate token, if ALICEO2_CCDB_AUTH_TOKEN names one, to a header list.
+///
+/// Set when CCDB is reached through a broker that authenticates its callers.
+/// The CI does this so the credential CCDB wants for writes -- a grid
+/// certificate -- stays in the broker and never enters the build container,
+/// which runs pull-request code. The broker consumes this header and does not
+/// forward it, so CCDB itself never sees it.
+///
+/// Read once into a static: getenv races setenv, and these paths run from
+/// several threads. Returns the list unchanged when no token is configured, so
+/// callers can apply it unconditionally.
+curl_slist* appendGateToken(curl_slist* list)
+{
+  static const std::string header = []() -> std::string {
+    const char* token = getenv("ALICEO2_CCDB_AUTH_TOKEN");
+    return (token && *token) ? std::string("Authorization: Bearer ") + token : std::string();
+  }();
+  return header.empty() ? list : curl_slist_append(list, header.c_str());
+}
+} // namespace
+
 /**
  * Object, encapsulating a semaphore, regulating
  * concurrent (multi-process) access to CCDB snapshot files.
@@ -428,6 +451,8 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin
     static const char buf[] = "Expect:";
     headerlist = curl_slist_append(headerlist, buf);
 
+    headerlist = appendGateToken(headerlist);
+
     curlSetSSLOptions(curl);
 
     curl_easy_setopt(curl, CURLOPT_MIMEPOST, mime);
@@ -722,6 +747,8 @@ void CcdbApi::initCurlHTTPHeaderOptionsForRetrieve(CURL* curlHandle, curl_slist*
     curl_easy_setopt(curlHandle, CURLOPT_HEADERDATA, headers);
   }
 
+  option_list = appendGateToken(option_list);
+
   if (option_list) {
     curl_easy_setopt(curlHandle, CURLOPT_HTTPHEADER, option_list);
   }
@@ -795,7 +822,7 @@ bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map<
 TObject* CcdbApi::retrieve(std::string const& path, std::map<std::string, std::string> const& metadata,
                            long timestamp) const
 {
-  struct MemoryStruct chunk {
+  struct MemoryStruct chunk{
     (char*)malloc(1) /*memory*/, 0 /*size*/
   };
 
@@ -1237,6 +1264,7 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string
     if (createdNotBefore >= 0) {
       headers = curl_slist_append(headers, ("If-Not-Before: " + std::to_string(createdNotBefore)).c_str());
     }
+    headers = appendGateToken(headers);
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
 
     curlSetSSLOptions(curl);
@@ -1453,6 +1481,7 @@ std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& p
     if (curl != nullptr) {
       struct curl_slist* list = nullptr;
       list = curl_slist_append(list, ("If-None-Match: " + std::to_string(timestamp)).c_str());
+      list = appendGateToken(list);
 
       curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
 
@@ -1531,6 +1560,7 @@ bool CcdbApi::getCCDBEntryHeaders(std::string const& url, std::string const& eta
 
   struct curl_slist* list = nullptr;
   list = curl_slist_append(list, ("If-None-Match: " + etag).c_str());
+  list = appendGateToken(list);
 
   curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
 

CCDB/test/testBasicCCDBManager.cxx

@@ -23,6 +23,7 @@
 #include "CCDB/BasicCCDBManager.h"
 #include "Framework/Logger.h"
 #include <boost/test/unit_test.hpp>
+#include <cstdlib>
 
 using namespace o2::ccdb;
 
@@ -37,6 +38,11 @@ struct Fixture {
   Fixture()
   {
     CcdbApi api;
+    // These suites upload, so they need a WRITABLE instance -- ccdb-test by
+    // default, not the official CCDB.
+    if (const char* host = std::getenv("ALICEO2_CCDB_HOST")) {
+      ccdbUrl = host;
+    }
     api.init(ccdbUrl);
     std::cout << "ccdb url: " << ccdbUrl << std::endl;
     hostReachable = api.isHostReachable();
@@ -134,7 +140,7 @@ BOOST_AUTO_TEST_CASE(TestBasicCCDBManager)
   BOOST_CHECK(objB && (*objB) == ccdbObjO); // make sure correct object is loaded
 
   // get object in TimeMachine mode in the past
-  cdb.setCreatedNotAfter(1);          // set upper object validity
+  cdb.setCreatedNotAfter(1); // set upper object validity
   cdb.setFatalWhenNull(false);
   objA = cdb.get<std::string>(pathA); // should not be loaded
   BOOST_CHECK(!objA);                 // make sure correct object is not loaded

CCDB/test/testCcdbApi.cxx

@@ -20,14 +20,15 @@
 #define BOOST_TEST_DYN_LINK
 
 #include "CCDB/CcdbApi.h"
-#include "CCDB/IdPath.h"    // just as test object
+#include "CCDB/IdPath.h"           // just as test object
 #include "CommonUtils/RootChain.h" // just as test object
 #include "CCDB/CCDBTimeStampUtils.h"
 #include <boost/test/unit_test.hpp>
 #include <filesystem>
 #include <iostream>
 #include <TH1F.h>
 #include <chrono>
+#include <cstdlib>
 #include <CommonUtils/StringUtils.h>
 #include <TStreamerInfo.h>
 #include <TGraph.h>
@@ -45,7 +46,7 @@ using namespace o2::ccdb;
 namespace utf = boost::unit_test;
 namespace tt = boost::test_tools;
 
-static std::string ccdbUrl;
+static std::string ccdbUrl = "http://ccdb-test.cern.ch:8080";
 static std::string basePath;
 bool hostReachable = false;
 
@@ -56,7 +57,11 @@ struct Fixture {
   Fixture()
   {
     CcdbApi api;
-    ccdbUrl = "http://ccdb-test.cern.ch:8080";
+    // These suites upload, so they need a WRITABLE instance -- ccdb-test by
+    // default, not the official CCDB.
+    if (const char* host = std::getenv("ALICEO2_CCDB_HOST")) {
+      ccdbUrl = host;
+    }
     api.init(ccdbUrl);
     cout << "ccdb url: " << ccdbUrl << endl;
     hostReachable = api.isHostReachable();
@@ -65,7 +70,7 @@ struct Fixture {
     gethostname(hostname, _POSIX_HOST_NAME_MAX);
     basePath = std::string("Test/TestCcdbApi/") + hostname + "/pid" + getpid() + "/";
     // Replace dashes by underscores to avoid problems in the creation of local directories
-    std::replace(basePath.begin(), basePath.end(), '-','_');
+    std::replace(basePath.begin(), basePath.end(), '-', '_');
     cout << "Path we will use in this test suite : " + basePath << endl;
   }
   ~Fixture()
@@ -446,13 +451,13 @@ BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable()))
   std::vector<std::string> headers;
   std::vector<std::string> pfns;
   std::string path = objectPath + "/" + std::to_string(getCurrentTimestamp());
-  auto updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/" + path, etag, headers);
+  auto updated = CcdbApi::getCCDBEntryHeaders(ccdbUrl + "/" + path, etag, headers);
   BOOST_CHECK_EQUAL(updated, true);
   BOOST_REQUIRE(headers.size() != 0);
   CcdbApi::parseCCDBHeaders(headers, pfns, etag);
   BOOST_REQUIRE(etag != "");
   BOOST_REQUIRE(pfns.size());
-  updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/" + path, etag, headers);
+  updated = CcdbApi::getCCDBEntryHeaders(ccdbUrl + "/" + path, etag, headers);
   BOOST_CHECK_EQUAL(updated, false);
 }
 
@@ -557,7 +562,7 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable()))
 BOOST_AUTO_TEST_CASE(multi_host_test)
 {
   CcdbApi api;
-  api.init("http://bogus-host.cern.ch,http://ccdb-test.cern.ch:8080");
+  api.init("http://bogus-host.cern.ch," + ccdbUrl);
   std::map<std::string, std::string> metadata;
   std::map<std::string, std::string> headers;
   o2::pmr::vector<char> dst;
@@ -569,7 +574,7 @@ BOOST_AUTO_TEST_CASE(multi_host_test)
 BOOST_AUTO_TEST_CASE(vectored)
 {
   CcdbApi api;
-  api.init("http://ccdb-test.cern.ch:8080");
+  api.init(ccdbUrl);
 
   int TEST_SAMPLE_SIZE = 5;
   std::vector<o2::pmr::vector<char>> dests(TEST_SAMPLE_SIZE);

CCDB/test/testCcdbApiHeaders.cxx

@@ -23,6 +23,7 @@
 #include "CCDB/CCDBTimeStampUtils.h"
 #include "CCDB/CcdbApi.h"
 #include <boost/test/unit_test.hpp>
+#include <cstdlib>
 
 static std::string basePath;
 // std::string ccdbUrl = "http://localhost:8080";
@@ -37,8 +38,10 @@ struct Fixture {
   Fixture()
   {
     auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance();
-    if (std::getenv("ALICEO2_CCDB_HOST")) {
-      ccdbUrl = std::string(std::getenv("ALICEO2_CCDB_HOST"));
+    // These suites upload, so they need a WRITABLE instance -- ccdb-test by
+    // default, not the official CCDB.
+    if (const char* host = std::getenv("ALICEO2_CCDB_HOST")) {
+      ccdbUrl = host;
     }
     ccdbManager.setURL(ccdbUrl);
     hostReachable = ccdbManager.getCCDBAccessor().isHostReachable();