- commit
- cda5bd5
- parent
- cae15d7
- author
- Eric Bower
- date
- 2025-06-20 14:57:12 -0400 EDT
refactor(config): remove old cfg options
8 files changed,
+6,
-344
+0,
-128
1@@ -1,128 +0,0 @@
2-package main
3-
4-import (
5- "log/slog"
6- "os"
7- "strings"
8-
9- "github.com/picosh/pico/pkg/apps/pgs"
10- pgsdb "github.com/picosh/pico/pkg/apps/pgs/db"
11- "github.com/picosh/pico/pkg/db"
12- "github.com/picosh/pico/pkg/shared"
13- "github.com/picosh/pico/pkg/shared/storage"
14- "github.com/picosh/utils"
15-)
16-
17-func bail(err error) {
18- if err != nil {
19- panic(err)
20- }
21-}
22-
23-type RmProject struct {
24- user *db.User
25- name string
26-}
27-
28-// this script will find any objects stored within Store that does not
29-// have a corresponding project inside our database.
30-func main() {
31- // to actually commit changes, set to true
32- writeEnv := utils.GetEnv("WRITE", "0")
33- write := writeEnv == "1"
34-
35- logger := slog.Default()
36-
37- picoCfg := shared.NewConfigSite()
38- picoCfg.Logger = logger
39- picoCfg.DbURL = os.Getenv("DATABASE_URL")
40- picoCfg.MinioURL = os.Getenv("MINIO_URL")
41- picoCfg.MinioUser = os.Getenv("MINIO_ROOT_USER")
42- picoCfg.MinioPass = os.Getenv("MINIO_ROOT_PASSWORD")
43- picoDb, err := pgsdb.NewDB(picoCfg.DbURL, picoCfg.Logger)
44- bail(err)
45-
46- var st storage.StorageServe
47- st, err = storage.NewStorageMinio(logger, picoCfg.MinioURL, picoCfg.MinioUser, picoCfg.MinioPass)
48- bail(err)
49-
50- logger.Info("fetching all users")
51- users, err := picoDb.FindUsers()
52- bail(err)
53-
54- logger.Info("fetching all buckets")
55- buckets, err := st.ListBuckets()
56- bail(err)
57-
58- rmProjects := []RmProject{}
59-
60- for _, bucketName := range buckets {
61- // only care about pgs
62- if !strings.HasPrefix(bucketName, "static-") {
63- continue
64- }
65-
66- bucket, err := st.GetBucket(bucketName)
67- bail(err)
68- bucketProjects, err := st.ListObjects(bucket, "/", false)
69- bail(err)
70-
71- userID := strings.Replace(bucketName, "static-", "", 1)
72- user := &db.User{
73- ID: userID,
74- Name: userID,
75- }
76- for _, u := range users {
77- if u.ID == userID {
78- user = u
79- break
80- }
81- }
82- projects, err := picoDb.FindProjectsByUser(userID)
83- bail(err)
84- for _, bucketProject := range bucketProjects {
85- found := false
86- for _, project := range projects {
87- // ignore links
88- if project.Name != project.ProjectDir {
89- continue
90- }
91- if project.Name == bucketProject.Name() {
92- found = true
93- }
94- }
95- if !found {
96- logger.Info("marking for removal", "bucket", bucketName, "project", bucketProject.Name())
97- rmProjects = append(rmProjects, RmProject{
98- name: bucketProject.Name(),
99- user: user,
100- })
101- }
102- }
103- }
104-
105- session := &utils.CmdSessionLogger{
106- Log: logger,
107- }
108-
109- for _, project := range rmProjects {
110- opts := &pgs.Cmd{
111- Session: session,
112- User: project.user,
113- Store: st,
114- Log: logger,
115- Dbpool: picoDb,
116- Write: write,
117- }
118- err := opts.RmProjectAssets(project.name)
119- bail(err)
120- }
121-
122- logger.Info("store projects marked for deletion", "length", len(rmProjects))
123- for _, project := range rmProjects {
124- logger.Info("removing project", "user", project.user.Name, "project", project.name)
125- }
126- if !write {
127- logger.Info("WARNING: changes not committed, need env var WRITE=1")
128- }
129-}
+0,
-102
1@@ -1,102 +0,0 @@
2-package main
3-
4-import (
5- "bytes"
6- "io"
7- "log/slog"
8- "path/filepath"
9- "time"
10-
11- "github.com/picosh/pico/pkg/apps/prose"
12- "github.com/picosh/pico/pkg/db"
13- "github.com/picosh/pico/pkg/db/postgres"
14- sst "github.com/picosh/pico/pkg/pobj/storage"
15- sendUtils "github.com/picosh/pico/pkg/send/utils"
16- "github.com/picosh/pico/pkg/shared"
17- "github.com/picosh/pico/pkg/shared/storage"
18-)
19-
20-func bail(err error) {
21- if err != nil {
22- panic(err)
23- }
24-}
25-
26-func upload(logger *slog.Logger, st storage.StorageServe, bucket sst.Bucket, fpath string, rdr io.Reader) error {
27- toSite := filepath.Join("prose", fpath)
28- logger.Info("uploading object", "bucket", bucket.Name, "object", toSite)
29- buf := &bytes.Buffer{}
30- size, err := io.Copy(buf, rdr)
31- if err != nil {
32- return err
33- }
34-
35- _, _, err = st.PutObject(bucket, toSite, buf, &sendUtils.FileEntry{
36- Mtime: time.Now().Unix(),
37- Size: size,
38- })
39- return err
40-}
41-
42-func images(logger *slog.Logger, dbh db.DB, st storage.StorageServe, bucket sst.Bucket, user *db.User) error {
43- posts, err := dbh.FindPostsForUser(&db.Pager{Num: 2000, Page: 0}, user.ID, "imgs")
44- if err != nil {
45- return err
46- }
47-
48- if len(posts.Data) == 0 {
49- logger.Info("user does not have any images, skipping")
50- return nil
51- }
52-
53- imgBucket, err := st.GetBucket(shared.GetImgsBucketName(user.ID))
54- if err != nil {
55- logger.Info("user does not have an images dir, skipping")
56- return nil
57- }
58-
59- /* imgs, err := st.ListObjects(imgBucket, "/", false)
60- if err != nil {
61- return err
62- } */
63-
64- for _, posts := range posts.Data {
65- rdr, _, err := st.GetObject(imgBucket, posts.Filename)
66- if err != nil {
67- logger.Error("get object", "err", err)
68- continue
69- }
70- err = upload(logger, st, bucket, posts.Filename, rdr)
71- if err != nil {
72- return err
73- }
74- }
75-
76- return nil
77-}
78-
79-func main() {
80- cfg := prose.NewConfigSite("prose-imgs-migrate")
81- logger := cfg.Logger
82- picoDb := postgres.NewDB(cfg.DbURL, logger)
83- st, err := storage.NewStorageMinio(logger, cfg.MinioURL, cfg.MinioUser, cfg.MinioPass)
84- bail(err)
85-
86- users, err := picoDb.FindUsers()
87- bail(err)
88-
89- for _, user := range users {
90- if user.Name != "erock" {
91- continue
92- }
93- logger.Info("migrating user images", "user", user.Name)
94-
95- bucket, err := st.UpsertBucket(shared.GetAssetBucketName(user.ID))
96- bail(err)
97- _, _ = picoDb.InsertProject(user.ID, "prose", "prose")
98- err = images(logger, picoDb, st, bucket, user)
99- if err != nil {
100- logger.Error("image uploader", "err", err)
101- }
102- }
103-}
1@@ -1,77 +0,0 @@
2-package main
3-
4-import (
5- "context"
6- "net/url"
7-
8- "github.com/minio/minio-go/v7"
9- "github.com/minio/minio-go/v7/pkg/credentials"
10- "github.com/picosh/pico/pkg/apps/prose"
11- "github.com/picosh/pico/pkg/db/postgres"
12- "github.com/picosh/pico/pkg/shared"
13-)
14-
15-func bail(err error) {
16- if err != nil {
17- panic(err)
18- }
19-}
20-
21-func main() {
22- cfg := prose.NewConfigSite("prose-rm-old-buckets")
23- logger := cfg.Logger
24- picoDb := postgres.NewDB(cfg.DbURL, logger)
25- endpoint, err := url.Parse(cfg.MinioURL)
26- bail(err)
27- ssl := endpoint.Scheme == "https"
28- mClient, err := minio.New(endpoint.Host, &minio.Options{
29- Creds: credentials.NewStaticV4(cfg.MinioUser, cfg.MinioPass, ""),
30- Secure: ssl,
31- })
32- bail(err)
33-
34- users, err := picoDb.FindUsers()
35- bail(err)
36- ctx := context.TODO()
37-
38- for _, user := range users {
39- logger.Info("deleting old buckets", "user", user.Name)
40- bucketName := shared.GetImgsBucketName(user.ID)
41-
42- exists, err := mClient.BucketExists(ctx, bucketName)
43- if err != nil {
44- logger.Error("bucket exists", "err", err)
45- }
46-
47- if !exists {
48- continue
49- }
50-
51- objectsCh := make(chan minio.ObjectInfo)
52- go func() {
53- defer close(objectsCh)
54- opts := minio.ListObjectsOptions{Prefix: "", Recursive: true}
55- for object := range mClient.ListObjects(ctx, bucketName, opts) {
56- logger.Info("object", "name", object.Key)
57- if object.Err != nil {
58- logger.Error("list objects", "err", err)
59- }
60- objectsCh <- object
61- }
62- }()
63-
64- errorCh := mClient.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{})
65-
66- for e := range errorCh {
67- logger.Error("remove obj", "err", e)
68- }
69-
70- logger.Info("removing bucket", "user", user.Name, "bucket", bucketName)
71- err = mClient.RemoveBucket(ctx, bucketName)
72- if err != nil {
73- logger.Error("remove bucket", "err", err)
74- }
75-
76- logger.Info("Success!", "user", user.Name)
77- }
78-}
+0,
-8
1@@ -10,10 +10,6 @@ func NewConfigSite(service string) *shared.ConfigSite {
2 domain := utils.GetEnv("FEEDS_DOMAIN", "feeds.pico.sh")
3 port := utils.GetEnv("FEEDS_WEB_PORT", "3000")
4 protocol := utils.GetEnv("FEEDS_PROTOCOL", "https")
5- storageDir := utils.GetEnv("IMGS_STORAGE_DIR", ".storage")
6- minioURL := utils.GetEnv("MINIO_URL", "")
7- minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
8- minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
9 dbURL := utils.GetEnv("DATABASE_URL", "")
10 sendgridKey := utils.GetEnv("SENDGRID_API_KEY", "")
11
12@@ -24,10 +20,6 @@ func NewConfigSite(service string) *shared.ConfigSite {
13 Port: port,
14 Protocol: protocol,
15 DbURL: dbURL,
16- StorageDir: storageDir,
17- MinioURL: minioURL,
18- MinioUser: minioUser,
19- MinioPass: minioPass,
20 Space: "feeds",
21 AllowedExt: []string{".txt"},
22 HiddenPosts: []string{"_header.txt", "_readme.txt"},
+0,
-8
1@@ -11,10 +11,6 @@ func NewConfigSite(service string) *shared.ConfigSite {
2 port := utils.GetEnv("PASTES_WEB_PORT", "3000")
3 dbURL := utils.GetEnv("DATABASE_URL", "")
4 protocol := utils.GetEnv("PASTES_PROTOCOL", "https")
5- storageDir := utils.GetEnv("IMGS_STORAGE_DIR", ".storage")
6- minioURL := utils.GetEnv("MINIO_URL", "")
7- minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
8- minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
9
10 return &shared.ConfigSite{
11 Debug: debug == "1",
12@@ -22,10 +18,6 @@ func NewConfigSite(service string) *shared.ConfigSite {
13 Port: port,
14 Protocol: protocol,
15 DbURL: dbURL,
16- StorageDir: storageDir,
17- MinioURL: minioURL,
18- MinioUser: minioUser,
19- MinioPass: minioPass,
20 Space: "pastes",
21 Logger: shared.CreateLogger(service),
22 MaxAssetSize: int64(3 * utils.MB),
+0,
-3
1@@ -20,7 +20,6 @@ type PgsConfig struct {
2 MaxSpecialFileSize int64
3 SshHost string
4 SshPort string
5- StorageDir string
6 TxtPrefix string
7 WebPort string
8 WebProtocol string
9@@ -71,7 +70,6 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
10 domain := utils.GetEnv("PGS_DOMAIN", "pgs.sh")
11 port := utils.GetEnv("PGS_WEB_PORT", "3000")
12 protocol := utils.GetEnv("PGS_PROTOCOL", "https")
13- storageDir := utils.GetEnv("PGS_STORAGE_DIR", ".storage")
14 cacheTTL, err := time.ParseDuration(utils.GetEnv("PGS_CACHE_TTL", ""))
15 if err != nil {
16 cacheTTL = 600 * time.Second
17@@ -92,7 +90,6 @@ func NewPgsConfig(logger *slog.Logger, dbpool pgsdb.PgsDB, st storage.StorageSer
18 MaxSpecialFileSize: maxSpecialFileSize,
19 SshHost: sshHost,
20 SshPort: sshPort,
21- StorageDir: storageDir,
22 TxtPrefix: "pgs",
23 WebPort: port,
24 WebProtocol: protocol,
+6,
-14
1@@ -12,25 +12,17 @@ func NewConfigSite(service string) *shared.ConfigSite {
2 domain := utils.GetEnv("PROSE_DOMAIN", "prose.sh")
3 port := utils.GetEnv("PROSE_WEB_PORT", "3000")
4 protocol := utils.GetEnv("PROSE_PROTOCOL", "https")
5- storageDir := utils.GetEnv("PROSE_STORAGE_DIR", ".storage")
6- minioURL := utils.GetEnv("MINIO_URL", "")
7- minioUser := utils.GetEnv("MINIO_ROOT_USER", "")
8- minioPass := utils.GetEnv("MINIO_ROOT_PASSWORD", "")
9 dbURL := utils.GetEnv("DATABASE_URL", "")
10 maxSize := uint64(25 * utils.MB)
11 maxImgSize := int64(10 * utils.MB)
12
13 return &shared.ConfigSite{
14- Debug: debug == "1",
15- Domain: domain,
16- Port: port,
17- Protocol: protocol,
18- DbURL: dbURL,
19- StorageDir: storageDir,
20- MinioURL: minioURL,
21- MinioUser: minioUser,
22- MinioPass: minioPass,
23- Space: "prose",
24+ Debug: debug == "1",
25+ Domain: domain,
26+ Port: port,
27+ Protocol: protocol,
28+ DbURL: dbURL,
29+ Space: "prose",
30 AllowedExt: []string{
31 ".md",
32 ".jpg",
1@@ -37,10 +37,6 @@ type ConfigSite struct {
2 PortOverride string
3 Protocol string
4 DbURL string
5- StorageDir string
6- MinioURL string
7- MinioUser string
8- MinioPass string
9 Space string
10 Issuer string
11 Secret string