Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ require (
github.com/onsi/gomega v1.42.1
github.com/open-policy-agent/opa v1.17.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260508132157-0223d2e98d28
github.com/opencloud-eu/reva/v2 v2.46.4-0.20260625152426-8cff2a7032ec
github.com/opensearch-project/opensearch-go/v4 v4.6.0
github.com/orcaman/concurrent-map v1.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -947,8 +947,8 @@ github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-202505121527
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a/go.mod h1:pjcozWijkNPbEtX5SIQaxEW/h8VAVZYTLx+70bmB3LY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+lP5lUUIzjRGDg93WrQfZJZCaV1ZP3KeyXi8bzY=
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260508132157-0223d2e98d28 h1:/sYlOEZav7FNOUwBMw99/VZ6QhljRCOBF6zsXWYE6O4=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260508132157-0223d2e98d28/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/reva/v2 v2.46.4-0.20260625152426-8cff2a7032ec h1:kL9P87cVpe8GKvo5vzLpJstpiVXFvkZn2+Dco53nfuM=
github.com/opencloud-eu/reva/v2 v2.46.4-0.20260625152426-8cff2a7032ec/go.mod h1:l4CG5u8tdWZj6IyavXO9xDdnNTV85oaFudGwEDPvsnk=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
Expand Down
275 changes: 189 additions & 86 deletions protogen/gen/opencloud/messages/search/v0/search.pb.go

Large diffs are not rendered by default.

36 changes: 36 additions & 0 deletions protogen/gen/opencloud/messages/search/v0/search.pb.web.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 20 additions & 0 deletions protogen/gen/opencloud/services/search/v0/search.swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,9 @@
"items": {
"type": "string"
}
},
"motionPhoto": {
"$ref": "#/definitions/v0MotionPhoto"
}
}
},
Expand Down Expand Up @@ -352,6 +355,23 @@
}
}
},
"v0MotionPhoto": {
"type": "object",
"properties": {
"version": {
"type": "integer",
"format": "int32"
},
"presentationTimestampUs": {
"type": "string",
"format": "int64"
},
"videoSize": {
"type": "string",
"format": "int64"
}
}
},
"v0Photo": {
"type": "object",
"properties": {
Expand Down
7 changes: 7 additions & 0 deletions protogen/proto/opencloud/messages/search/v0/search.proto
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ message Photo {
optional google.protobuf.Timestamp takenDateTime = 9;
}

message MotionPhoto {
optional int32 version = 1;
optional int64 presentationTimestampUs = 2;
optional int64 videoSize = 3;
}

message Entity {
Reference ref = 1;
ResourceID id = 2;
Expand All @@ -80,6 +86,7 @@ message Entity {
Image image = 18;
Photo photo = 19;
repeated string favorites = 20;
MotionPhoto motionPhoto = 21;
}

message Match {
Expand Down
1 change: 1 addition & 0 deletions services/graph/pkg/service/v0/driveitems.go
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,7 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
driveItem.LibreGraphMotionPhoto = metadataToFacet[libregraph.MotionPhoto](metadata, "motionPhoto")
}

return driveItem, nil
Expand Down
29 changes: 15 additions & 14 deletions services/search/pkg/bleve/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,20 +126,21 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
ResourceId: resourceIDtoSearchID(rootID),
Path: getFieldValue[string](hit.Fields, "Path"),
},
Id: resourceIDtoSearchID(rID),
Name: getFieldValue[string](hit.Fields, "Name"),
ParentId: resourceIDtoSearchID(pID),
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
Id: resourceIDtoSearchID(rID),
Name: getFieldValue[string](hit.Fields, "Name"),
ParentId: resourceIDtoSearchID(pID),
Size: uint64(getFieldValue[float64](hit.Fields, "Size")),
Type: uint64(getFieldValue[float64](hit.Fields, "Type")),
MimeType: getFieldValue[string](hit.Fields, "MimeType"),
Deleted: getFieldValue[bool](hit.Fields, "Deleted"),
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
MotionPhoto: hitToFacet[searchMessage.MotionPhoto](hit.Fields, "motionPhoto"),
},
}

Expand Down
25 changes: 13 additions & 12 deletions services/search/pkg/content/content.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,19 @@ func init() {
// Document wraps all resource meta fields,
// it is used as a content extraction result.
type Document struct {
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime string `json:"Mtime"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
Photo *libregraph.Photo `json:"photo,omitempty"`
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime string `json:"Mtime"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
Photo *libregraph.Photo `json:"photo,omitempty"`
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
}

func CleanString(content, langCode string) string {
Expand Down
72 changes: 65 additions & 7 deletions services/search/pkg/content/cs3.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,26 @@ func newCS3Retriever(gatewaySelector pool.Selectable[gateway.GatewayAPIClient],
}
}

// Retrieve downloads the file from a cs3 service
// The caller MUST make sure to close the returned ReadCloser
func (s cs3) Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadCloser, error) {
// initiateDownload asks the gateway for a download endpoint and returns it
// together with the transfer token and the auth token from the context.
func (s cs3) initiateDownload(ctx context.Context, rID *provider.ResourceId) (endpoint, transferToken, authToken string, err error) {
at, ok := contextGet(ctx, revactx.TokenHeader)
if !ok {
return nil, fmt.Errorf("context without %s", revactx.TokenHeader)
return "", "", "", fmt.Errorf("context without %s", revactx.TokenHeader)
}

gatewayClient, err := s.gatewaySelector.Next()
if err != nil {
s.logger.Error().Err(err).Msg("could not get reva gatewayClient")
return nil, err
return "", "", "", err
}

res, err := gatewayClient.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &provider.Reference{ResourceId: rID, Path: "."}})
if err != nil {
return nil, err
return "", "", "", err
}
if res.Status.Code != rpc.Code_CODE_OK {
return nil, fmt.Errorf("could not load resoure: %s", res.Status.Message)
return "", "", "", fmt.Errorf("could not load resoure: %s", res.Status.Message)
}

var ep, tt string
Expand All @@ -66,6 +66,17 @@ func (s cs3) Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadClo
ep, tt = res.Protocols[0].DownloadEndpoint, res.Protocols[0].Token
}

return ep, tt, at, nil
}

// Retrieve downloads the file from a cs3 service
// The caller MUST make sure to close the returned ReadCloser
func (s cs3) Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadCloser, error) {
ep, tt, at, err := s.initiateDownload(ctx, rID)
if err != nil {
return nil, err
}

req, err := http.NewRequest(http.MethodGet, ep, nil)
if err != nil {
return nil, err
Expand All @@ -80,8 +91,55 @@ func (s cs3) Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadClo
}

if cres.StatusCode != http.StatusOK {
_ = cres.Body.Close()
return nil, fmt.Errorf("could not download resource. Request returned with statuscode %d ", cres.StatusCode)
}

return cres.Body, nil
}

// RetrieveRange downloads length bytes starting at offset from a cs3 service.
// The caller MUST make sure to close the returned ReadCloser.
// It relies on HTTP range support of the download endpoint. If the endpoint
// ignores the Range header and returns the full file (200 instead of 206), the
// leading offset bytes are discarded so the returned reader is always positioned
// at offset.
func (s cs3) RetrieveRange(ctx context.Context, rID *provider.ResourceId, offset, length int64) (io.ReadCloser, error) {
ep, tt, at, err := s.initiateDownload(ctx, rID)
if err != nil {
return nil, err
}

req, err := http.NewRequest(http.MethodGet, ep, nil)
if err != nil {
return nil, err
}

req.Header.Set(revactx.TokenHeader, at)
req.Header.Set("X-Reva-Transfer", tt)
// A single range keeps the response a plain 206 with a Content-Range header,
// never a multipart/byteranges body.
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))

cres, err := s.httpClient.Do(req)
if err != nil {
return nil, err
}

switch cres.StatusCode {
case http.StatusPartialContent:
// Range honored: the body already starts at offset.
return cres.Body, nil
case http.StatusOK:
// Range ignored: the body is the whole file. Skip to offset so the
// caller always reads from the requested position.
if _, err := io.CopyN(io.Discard, cres.Body, offset); err != nil {
_ = cres.Body.Close()
return nil, fmt.Errorf("could not skip to offset %d: %w", offset, err)
}
return cres.Body, nil
default:
_ = cres.Body.Close()
return nil, fmt.Errorf("could not download range. Request returned with statuscode %d ", cres.StatusCode)
}
}
17 changes: 8 additions & 9 deletions services/search/pkg/content/extractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,18 @@ type Extractor interface {
Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, error)
}

func getFirstValue(m map[string][]string, key string) (string, error) {
// getFirstValue returns the first metadata value present among keys, trying them
// in order. It errors when the map is nil or none of the keys holds a value.
func getFirstValue(m map[string][]string, keys ...string) (string, error) {
if m == nil {
return "", errors.New("undefined map")
}

v, ok := m[key]
if !ok {
return "", fmt.Errorf("unknown key: %v", key)
for _, key := range keys {
if v, ok := m[key]; ok && len(v) > 0 {
return v[0], nil
}
}

if len(m) == 0 {
return "", fmt.Errorf("no values for: %v", key)
}

return v[0], nil
return "", fmt.Errorf("no value for keys: %v", keys)
}
Loading