Push based thumbnailer architecture

.codacy.yml

@@ -20,6 +20,9 @@ exclude_paths:
   - 'tests/acceptance/expected-failures-*.md'
   # written by the search engine parity suite, table rows are as wide as they are
   - 'services/search/pkg/parity/README.md'
+  # thumbnail docs: wide tables and prose that is not hard-wrapped
+  - 'services/thumbnails/README.md'
+  - 'services/webdav/README.md'
   - 'tests/acceptance/bootstrap/**'
   - 'tests/acceptance/TestHelpers/**'
   - 'tests/acceptance/scripts/run.sh'

.vscode/launch.json

@@ -162,8 +162,8 @@
         "STORE_DEBUG_ADDR": "127.0.0.1:10464",
         "STORE_GRPC_ADDR": "127.0.0.1:10460",
         "THUMBNAILS_DEBUG_ADDR": "127.0.0.1:10194",
-        "THUMBNAILS_GRPC_ADDR": "127.0.0.1:10191",
         "THUMBNAILS_HTTP_ADDR": "127.0.0.1:10190",
+        "WEBDAV_THUMBNAILS_GENERATOR_URL": "http://127.0.0.1:10190",
         "USERLOG_HTTP_ADDR": "127.0.0.1:10211",
         "USERLOG_DEBUG_ADDR": "127.0.0.1:10210",
         "USERS_DEBUG_ADDR": "127.0.0.1:10145",

.woodpecker.star

@@ -13,6 +13,10 @@ CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1"
 CODACY_COVERAGE_REPORTER = "codacy/codacy-coverage-reporter:14.1.3"
 COLLABORA_CODE = "collabora/code:24.04.5.1.1"
 OPEN_SEARCH = "opensearchproject/opensearch:2"
+
+# imagor for the thumbnail parity suite, keep equal to imagortest.Variants
+IMAGOR = "shumc/imagor:1.9.11"
+IMAGOR_MAGICK = "ghcr.io/cshum/imagor-magick:1.9.11"
 INBUCKET_INBUCKET = "inbucket/inbucket"
 MINIO_MC = "quay.io/opencloudeu/minio-mc-ci:1.0"
 OC_CI_ALPINE = "owncloudci/alpine:latest"
@@ -786,18 +790,34 @@ def testOpencloud(ctx):
             ],
             "environment": CI_HTTP_PROXY_ENV,
         },
-    ] + waitForOpenSearch() + [
+    ] + waitForOpenSearch() + waitForImagor() + [
         {
             "name": "test",
             "image": OC_CI_GOLANG,
             "environment": {
                 "SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES": "http://open-search:9200",
+                "THUMBNAILS_PARITY_IMAGOR_URL": "http://imagor:8000",
+                "THUMBNAILS_PARITY_IMAGOR_MAGICK_URL": "http://imagor-magick:8000",
             },
             "commands": [
                 "make test",
                 "mv coverage.out unit.coverage.out",
             ],
         },
+        {
+            # the only run that covers the resize code the official docker
+            # images ship
+            # FIXME: run it with the libvips those images ship, see issue 3746
+            "name": "test-thumbnail-parity-vips",
+            "image": OC_CI_GOLANG,
+            "environment": {
+                "THUMBNAILS_PARITY_IMAGOR_URL": "http://imagor:8000",
+                "THUMBNAILS_PARITY_IMAGOR_MAGICK_URL": "http://imagor-magick:8000",
+            },
+            "commands": [
+                "go test -tags enable_vips ./services/thumbnails/pkg/parity/...",
+            ],
+        },
         {
             "name": "codacy-coverage",
             "image": CODACY_COVERAGE_REPORTER,
@@ -818,7 +838,7 @@ def testOpencloud(ctx):
     pipeline = {
         "name": "test-lint-unit",
         "steps": steps,
-        "services": openSearchService(),
+        "services": openSearchService() + imagorServices(),
         "when": [
             event["base"],
             event["cron"],
@@ -3424,6 +3444,53 @@ def waitForOpenSearch():
         ],
     }]
 
+def imagorServices():
+    environment = {
+        # the parity suite posts every image, and there is nothing to sign with
+        "IMAGOR_UNSAFE": True,
+        "UPLOAD_LOADER_ENABLE": True,
+        "HTTP_LOADER_DISABLE": True,
+        # the limits of the thumbnails service, so both sides refuse the same
+        # requests; keep in step with imagortest.Args, which configures imagor
+        # the same way when the suite starts it through testcontainers
+        "VIPS_MAX_WIDTH": 7680,
+        "VIPS_MAX_HEIGHT": 7680,
+        "VIPS_MAX_ANIMATION_FRAMES": 100,
+        # quoted: a number this large reaches the container as 5e+07, which imagor cannot parse
+        "UPLOAD_LOADER_MAX_ALLOWED_SIZE": "50000000",
+        "IMAGOR_PROCESS_CONCURRENCY": -1,
+    }
+
+    return [
+        {
+            "name": "imagor",
+            "image": IMAGOR,
+            "environment": environment,
+        },
+        {
+            "name": "imagor-magick",
+            "image": IMAGOR_MAGICK,
+            "environment": environment,
+        },
+    ]
+
+def waitForImagor():
+    return [{
+        "name": "wait-for-imagor",
+        "image": OC_CI_ALPINE,
+        "commands": [
+            "bash -c '" +
+            "for host in imagor imagor-magick; do\n" +
+            "  until curl -sS -o /dev/null \"http://$host:8000/\"; do\n" +
+            "    echo \"Waiting for http://$host:8000 to be healthy...\"\n" +
+            "    sleep 2\n" +
+            "  done\n" +
+            "  echo \"http://$host:8000 healthy...\"\n" +
+            "done\n" +
+            "'",
+        ],
+    }]
+
 def logRequests():
     return [{
         "name": "api-test-failure-logs",

go.mod

@@ -55,6 +55,7 @@ require (
 	github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69
 	github.com/libregraph/idm v0.5.0
 	github.com/libregraph/lico v0.67.0
+	github.com/minio/minio-go/v7 v7.2.1
 	github.com/mna/pigeon v1.3.0
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/nats-io/nats-server/v2 v2.15.0
@@ -67,7 +68,7 @@ require (
 	github.com/open-policy-agent/opa v1.21.1
 	github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
 	github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346
-	github.com/opencloud-eu/reva/v2 v2.51.1-0.20261021052306-7983cec1cd3a
+	github.com/opencloud-eu/reva/v2 v2.51.1-0.20261023103014-40a03d16bc92
 	github.com/opensearch-project/opensearch-go/v4 v4.7.3
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/pkg/errors v0.9.1
@@ -282,7 +283,6 @@ require (
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/highwayhash v1.0.4 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.2.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect

go.sum

@@ -761,8 +761,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
 github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
 github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 h1:3ptbiFhSK4fzY/rC3idcYUveiwepMoOov1sBpc7Rg3k=
 github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
-github.com/opencloud-eu/reva/v2 v2.51.1-0.20261021052306-7983cec1cd3a h1:gChg4oBA5YYcpFrjKrlDXPjdOL5E6HknMSn8yj3vftc=
-github.com/opencloud-eu/reva/v2 v2.51.1-0.20261021052306-7983cec1cd3a/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8=
+github.com/opencloud-eu/reva/v2 v2.51.1-0.20261023103014-40a03d16bc92 h1:vGNerJ4b6uAmD8vBK9z1VYOYei+Xxs+K46GIFkQyZxM=
+github.com/opencloud-eu/reva/v2 v2.51.1-0.20261023103014-40a03d16bc92/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8=
 github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
 github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=

opencloud/pkg/init/init.go

@@ -68,7 +68,7 @@ func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword
 		systemUserID, adminUserID, graphApplicationID, storageUsersMountID, serviceAccountID string
 		idmServicePassword, idpServicePassword, ocAdminServicePassword, revaServicePassword  string
 		tokenManagerJwtSecret, collaborationWOPISecret, machineAuthAPIKey, systemUserAPIKey  string
-		revaTransferSecret, thumbnailsTransferSecret, serviceAccountSecret, urlSigningSecret string
+		revaTransferSecret, serviceAccountSecret, urlSigningSecret string
 		adminPasswdwordGenerated                                                             bool
 	)
 
@@ -94,7 +94,6 @@ func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword
 		machineAuthAPIKey = oldCfg.MachineAuthAPIKey
 		systemUserAPIKey = oldCfg.SystemUserAPIKey
 		revaTransferSecret = oldCfg.TransferSecret
-		thumbnailsTransferSecret = oldCfg.Thumbnails.Thumbnail.TransferSecret
 		serviceAccountSecret = oldCfg.Graph.ServiceAccount.ServiceAccountSecret
 		urlSigningSecret = oldCfg.URLSigningSecret
 		if urlSigningSecret == "" {
@@ -155,10 +154,6 @@ func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword
 		if err != nil {
 			return fmt.Errorf("could not generate random secret for urlSigningSecret: %s", err)
 		}
-		thumbnailsTransferSecret, err = generators.GenerateRandomPassword(passwordLength)
-		if err != nil {
-			return fmt.Errorf("could not generate random password for thumbnailsTransferSecret: %s", err)
-		}
 		serviceAccountSecret, err = generators.GenerateRandomPassword(passwordLength)
 		if err != nil {
 			return fmt.Errorf("could not generate random secret for serviceAccountSecret: %s", err)
@@ -233,11 +228,6 @@ func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword
 			},
 			ServiceAccount: serviceAccount,
 		},
-		Thumbnails: ThumbnailService{
-			Thumbnail: ThumbnailSettings{
-				TransferSecret: thumbnailsTransferSecret,
-			},
-		},
 		Gateway: Gateway{
 			StorageRegistry: StorageRegistry{
 				StorageUsersMountID: storageUsersMountID,
@@ -305,9 +295,6 @@ func CreateConfig(insecure, forceOverwrite, diff bool, configPath, adminPassword
 			},
 			ServiceAccount: serviceAccount,
 		}
-
-		cfg.Thumbnails.Thumbnail.WebdavAllowInsecure = true
-		cfg.Thumbnails.Thumbnail.Cs3AllowInsecure = true
 	}
 	yamlOutput, err := yaml.Marshal(cfg)
 	if err != nil {

opencloud/pkg/init/structs.go

@@ -34,7 +34,6 @@ type OpenCloudConfig struct {
 	Groups            UsersAndGroupsService `yaml:"groups"`
 	AuthGuest         AuthGuest             `yaml:"auth_guest"`
 	Ocm               OcmService            `yaml:"ocm"`
-	Thumbnails        ThumbnailService      `yaml:"thumbnails"`
 	Search            Search                `yaml:"search"`
 	Audit             Audit                 `yaml:"audit"`
 	Settings          SettingsService       `yaml:"settings"`
@@ -226,18 +225,6 @@ type StorageUsers struct {
 	ServiceAccount ServiceAccount `yaml:"service_account"`
 }
 
-// ThumbnailSettings is the configuration for the thumbnail settings
-type ThumbnailSettings struct {
-	TransferSecret      string `yaml:"transfer_secret"`
-	WebdavAllowInsecure bool   `yaml:"webdav_allow_insecure"`
-	Cs3AllowInsecure    bool   `yaml:"cs3_allow_insecure"`
-}
-
-// ThumbnailService is the configuration for the thumbnail service
-type ThumbnailService struct {
-	Thumbnail ThumbnailSettings
-}
-
 // TokenManager is the configuration for the token manager
 type TokenManager struct {
 	JWTSecret string `yaml:"jwt_secret"`

protogen/.mockery.yaml

@@ -19,6 +19,3 @@ packages:
   github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/settings/v0:
     interfaces:
       ValueService: {}
-  github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/thumbnails/v0:
-    interfaces:
-      ThumbnailService: {}