github-featured/apache--answer
2
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat(service): implement file cleanup and deletion functionality

Browse Source
This commit is contained in:
LinkinStars
2025-01-10 10:43:35 +08:00
parent 47c666200e
commit 5cffab8649
24 changed files with 505 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/wire_gen.go
+8 -4
View File
@@ -48,6 +48,7 @@ import (
"github.com/apache/answer/internal/repo/comment"
"github.com/apache/answer/internal/repo/config"
"github.com/apache/answer/internal/repo/export"
"github.com/apache/answer/internal/repo/file_record"
"github.com/apache/answer/internal/repo/limit"
"github.com/apache/answer/internal/repo/meta"
notification2 "github.com/apache/answer/internal/repo/notification"
@@ -84,6 +85,7 @@ import (
"github.com/apache/answer/internal/service/dashboard"
"github.com/apache/answer/internal/service/event_queue"
export2 "github.com/apache/answer/internal/service/export"
file_record2 "github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/service/follow"
"github.com/apache/answer/internal/service/importer"
meta2 "github.com/apache/answer/internal/service/meta"
@@ -227,10 +229,10 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
contentRevisionService := content.NewRevisionService(revisionRepo, userCommon, questionCommon, answerService, objService, questionRepo, answerRepo, tagRepo, tagCommonService, notificationQueueService, activityQueueService, reportRepo, reviewService, reviewActivityRepo)
revisionController := controller.NewRevisionController(contentRevisionService, rankService)
rankController := controller.NewRankController(rankService)
badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
notificationRepo := notification2.NewNotificationRepo(dataData)
pluginUserConfigRepo := plugin_config.NewPluginUserConfigRepo(dataData)
userAdminRepo := user.NewUserAdminRepo(dataData, authRepo)
badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo)
userAdminService := user_admin.NewUserAdminService(userAdminRepo, userRoleRelService, authService, userCommon, userActiveActivityRepo, siteInfoCommonService, emailService, questionRepo, answerRepo, commentCommonRepo, userExternalLoginRepo, notificationRepo, pluginUserConfigRepo, badgeAwardRepo)
userAdminController := controller_admin.NewUserAdminController(userAdminService)
reasonRepo := reason.NewReasonRepo(configService)
@@ -246,7 +248,9 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
notificationController := controller.NewNotificationController(notificationService, rankService)
dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData)
dashboardController := controller.NewDashboardController(dashboardService)
uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService)
fileRecordRepo := file_record.NewFileRecordRepo(dataData)
fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService)
uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService)
uploadController := controller.NewUploadController(uploaderService)
activityActivityRepo := activity.NewActivityRepo(dataData, configService)
activityCommon := activity_common2.NewActivityCommon(activityRepo, activityQueueService)
@@ -287,7 +291,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database,
renderController := controller.NewRenderController()
pluginAPIRouter := router.NewPluginAPIRouter(connectorController, userCenterController, captchaController, embedController, renderController)
ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, shortIDMiddleware, templateRouter, pluginAPIRouter, uiConf)
scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService)
scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService, fileRecordService, serviceConf)
application := newApplication(serverConf, ginEngine, scheduledTaskManager)
return application, func() {
cleanup2()
configs/config.yaml
+3
View File
@@ -33,6 +33,9 @@ swaggerui:
address: ':80'
service_config:
upload_path: "/data/uploads"
clean_up_uploads: true
clean_orphan_uploads_period_hours: 48
purge_deleted_files_period_days: 30
ui:
public_url: '/'
api_url: '/'
go.mod
+1 -1
View File
@@ -60,6 +60,7 @@ require (
golang.org/x/crypto v0.27.0
golang.org/x/image v0.20.0
golang.org/x/net v0.29.0
golang.org/x/text v0.18.0
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.33.0
@@ -160,7 +161,6 @@ require (
golang.org/x/arch v0.10.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/tools v0.25.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
internal/base/constant/upload.go
+1
View File
@@ -25,4 +25,5 @@ const (
PostSubPath = "post"
BrandingSubPath = "branding"
FilesPostSubPath = "files/post"
DeletedSubPath = "deleted"
)
internal/base/cron/cron.go
+36 -7
View File
@@ -24,6 +24,8 @@ import (
"fmt"
"github.com/apache/answer/internal/service/content"
"github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/service/service_config"
"github.com/apache/answer/internal/service/siteinfo_common"
"github.com/robfig/cron/v3"
"github.com/segmentfault/pacman/log"
@@ -31,29 +33,36 @@ import (
// ScheduledTaskManager scheduled task manager
type ScheduledTaskManager struct {
siteInfoService siteinfo_common.SiteInfoCommonService
questionService *content.QuestionService
siteInfoService siteinfo_common.SiteInfoCommonService
questionService *content.QuestionService
fileRecordService *file_record.FileRecordService
serviceConfig *service_config.ServiceConfig
}
// NewScheduledTaskManager new scheduled task manager
func NewScheduledTaskManager(
siteInfoService siteinfo_common.SiteInfoCommonService,
questionService *content.QuestionService,
fileRecordService *file_record.FileRecordService,
serviceConfig *service_config.ServiceConfig,
) *ScheduledTaskManager {
manager := &ScheduledTaskManager{
siteInfoService: siteInfoService,
questionService: questionService,
siteInfoService: siteInfoService,
questionService: questionService,
fileRecordService: fileRecordService,
serviceConfig: serviceConfig,
}
return manager
}
func (s *ScheduledTaskManager) Run() {
fmt.Println("start cron")
log.Infof("cron job manager start")
s.questionService.SitemapCron(context.Background())
c := cron.New()
_, err := c.AddFunc("0 */1 * * *", func() {
ctx := context.Background()
fmt.Println("sitemap cron execution")
log.Infof("sitemap cron execution")
s.questionService.SitemapCron(ctx)
})
if err != nil {
@@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() {
_, err = c.AddFunc("0 */1 * * *", func() {
ctx := context.Background()
fmt.Println("refresh hottest cron execution")
log.Infof("refresh hottest cron execution")
s.questionService.RefreshHottestCron(ctx)
})
if err != nil {
log.Error(err)
}
if s.serviceConfig.CleanUpUploads {
log.Infof("clean up uploads cron enabled")
conf := s.serviceConfig
_, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", conf.CleanOrphanUploadsPeriodHours), func() {
log.Infof("clean orphan upload files cron execution")
s.fileRecordService.CleanOrphanUploadFiles(context.Background())
})
if err != nil {
log.Error(err)
}
_, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", conf.PurgeDeletedFilesPeriodDays), func() {
log.Infof("purge deleted files cron execution")
s.fileRecordService.PurgeDeletedFiles(context.Background())
})
if err != nil {
log.Error(err)
}
}
c.Start()
}
internal/controller/upload_controller.go
+5 -4
View File
@@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) {
)
source := ctx.PostForm("source")
userID := middleware.GetLoginUserIDFromContext(ctx)
switch source {
case fileFromAvatar:
url, err = uc.uploaderService.UploadAvatarFile(ctx)
url, err = uc.uploaderService.UploadAvatarFile(ctx, userID)
case fileFromPost:
url, err = uc.uploaderService.UploadPostFile(ctx)
url, err = uc.uploaderService.UploadPostFile(ctx, userID)
case fileFromBranding:
if !middleware.GetIsAdminFromContext(ctx) {
handler.HandleResponse(ctx, errors.Forbidden(reason.ForbiddenError), nil)
return
}
url, err = uc.uploaderService.UploadBrandingFile(ctx)
url, err = uc.uploaderService.UploadBrandingFile(ctx, userID)
case fileFromPostAttachment:
url, err = uc.uploaderService.UploadPostAttachment(ctx)
url, err = uc.uploaderService.UploadPostAttachment(ctx, userID)
default:
handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil)
return
internal/entity/file_record_entity.go
+45
View File
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package entity
import "time"
const (
FileRecordStatusAvailable = 1
FileRecordStatusDeleted = 10
)
// FileRecord file record
type FileRecord struct {
ID int `xorm:"not null pk autoincr INT(10) id"`
CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created TIMESTAMP created_at"`
UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated TIMESTAMP updated_at"`
UserID string `xorm:"not null default 0 BIGINT(20) user_id"`
FilePath string `xorm:"not null VARCHAR(256) file_path"`
FileURL string `xorm:"not null VARCHAR(1024) file_url"`
ObjectID string `xorm:"not null default 0 INDEX BIGINT(20) object_id"`
Source string `xorm:"not null VARCHAR(128) source"`
Status int `xorm:"not null default 0 TINYINT(4) status"`
}
// TableName file record table name
func (FileRecord) TableName() string {
return "file_record"
}
internal/migrations/migrations.go
+1
View File
@@ -100,6 +100,7 @@ var migrations = []Migration{
NewMigration("v1.4.0", "add badge/badge_group/badge_award table", addBadges, true),
NewMigration("v1.4.1", "add question link", addQuestionLink, true),
NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true),
NewMigration("v1.4.5", "add file record", addFileRecord, true),
}
func GetMigrations() []Migration {
internal/migrations/v25.go
+31
View File
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package migrations
import (
"context"
"github.com/apache/answer/internal/entity"
"xorm.io/xorm"
)
func addFileRecord(ctx context.Context, x *xorm.Engine) error {
return x.Context(ctx).Sync(new(entity.FileRecord))
}
internal/repo/answer/answer_repo.go
+18 -1
View File
@@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, answerID string) (err er
}
func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error {
_, err := ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
// get all deleted answers ids
ids := make([]string, 0)
err := ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()).
Where("status = ?", entity.AnswerStatusDeleted).Find(&ids)
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
if len(ids) == 0 {
return nil
}
// delete all revisions permanently
_, err = ar.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
_, err = ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
internal/repo/file_record/file_record_repo.go
+84
View File
@@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package file_record
import (
"context"
"github.com/apache/answer/internal/base/pager"
"github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/base/data"
"github.com/apache/answer/internal/base/reason"
"github.com/apache/answer/internal/entity"
"github.com/segmentfault/pacman/errors"
)
// fileRecordRepo fileRecord repository
type fileRecordRepo struct {
data *data.Data
}
// NewFileRecordRepo new repository
func NewFileRecordRepo(data *data.Data) file_record.FileRecordRepo {
return &fileRecordRepo{
data: data,
}
}
// AddFileRecord add file record
func (fr *fileRecordRepo) AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {
_, err = fr.data.DB.Context(ctx).Insert(fileRecord)
if err != nil {
err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
}
// GetFileRecordPage get fileRecord page
func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (
fileRecordList []*entity.FileRecord, total int64, err error) {
fileRecordList = make([]*entity.FileRecord, 0)
session := fr.data.DB.Context(ctx)
total, err = pager.Help(page, pageSize, &fileRecordList, cond, session)
if err != nil {
err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
}
// DeleteFileRecord delete file record
func (fr *fileRecordRepo) DeleteFileRecord(ctx context.Context, id int) (err error) {
_, err = fr.data.DB.Context(ctx).ID(id).Cols("status").Update(&entity.FileRecord{Status: entity.FileRecordStatusDeleted})
if err != nil {
err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
}
// UpdateFileRecord update file record
func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) {
_, err = fr.data.DB.Context(ctx).ID(fileRecord.ID).Update(fileRecord)
if err != nil {
err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
}
internal/repo/provider.go
+2
View File
@@ -33,6 +33,7 @@ import (
"github.com/apache/answer/internal/repo/comment"
"github.com/apache/answer/internal/repo/config"
"github.com/apache/answer/internal/repo/export"
"github.com/apache/answer/internal/repo/file_record"
"github.com/apache/answer/internal/repo/limit"
"github.com/apache/answer/internal/repo/meta"
"github.com/apache/answer/internal/repo/notification"
@@ -107,4 +108,5 @@ var ProviderSetRepo = wire.NewSet(
badge.NewEventRuleRepo,
badge_group.NewBadgeGroupRepo,
badge_award.NewBadgeAwardRepo,
file_record.NewFileRecordRepo,
)
internal/repo/question/question_repo.go
+17
View File
@@ -168,6 +168,23 @@ func (qr *questionRepo) UpdateQuestionStatusWithOutUpdateTime(ctx context.Contex
}
func (qr *questionRepo) DeletePermanentlyQuestions(ctx context.Context) (err error) {
// get all deleted question ids
ids := make([]string, 0)
err = qr.data.DB.Context(ctx).Select("id").Table(new(entity.Question).TableName()).
Where("status = ?", entity.QuestionStatusDeleted).Find(&ids)
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
if len(ids) == 0 {
return nil
}
// delete all revisions permanently
_, err = qr.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
_, err = qr.data.DB.Context(ctx).Where("status = ?", entity.QuestionStatusDeleted).Delete(&entity.Question{})
if err != nil {
return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
internal/repo/revision/revision_repo.go
+11 -1
View File
@@ -155,7 +155,17 @@ func (rr *revisionRepo) GetLastRevisionByObjectID(ctx context.Context, objectID
revision *entity.Revision, exist bool, err error,
) {
revision = &entity.Revision{}
exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).OrderBy("created_at DESC").Get(revision)
exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).Desc("created_at").Get(revision)
if err != nil {
return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
return
}
// GetLastRevisionByFileURL get object's last revision by file url
func (rr *revisionRepo) GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error) {
revision = &entity.Revision{}
exist, err = rr.data.DB.Context(ctx).Where("content LIKE ?", "%"+fileURL+"%").Desc("created_at").Get(revision)
if err != nil {
return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
}
internal/router/static_router.go
+11 -3
View File
@@ -20,11 +20,14 @@
package router
import (
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/service/service_config"
"github.com/gin-gonic/gin"
"net/http"
"path/filepath"
"strings"
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/service/service_config"
"github.com/apache/answer/pkg/dir"
"github.com/gin-gonic/gin"
)
// StaticRouter static api router
@@ -54,6 +57,11 @@ func (a *StaticRouter) RegisterStaticRouter(r *gin.RouterGroup) {
realFilename := strings.TrimSuffix(filePath, "/"+originalFilename) + filepath.Ext(originalFilename)
// The file local path is /uploads/files/post/hash.pdf
fileLocalPath := filepath.Join(a.serviceConfig.UploadPath, constant.FilesPostSubPath, realFilename)
// If the file is not exist, return 404
if !dir.CheckFileExist(fileLocalPath) {
c.Redirect(http.StatusFound, "/404")
return
}
c.FileAttachment(fileLocalPath, originalFilename)
})
}
internal/service/content/question_service.go
+2
View File
@@ -1398,6 +1398,8 @@ func (qs *QuestionService) GetQuestionPage(ctx context.Context, req *schema.Ques
return nil, 0, err
}
tagIDs = append(synTagIds, tagInfo.ID)
} else {
return questions, 0, nil
}
}
internal/service/file_record/file_record_service.go
+172
View File
@@ -0,0 +1,172 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package file_record
import (
"context"
"fmt"
"os"
"path/filepath"
"time"
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/entity"
"github.com/apache/answer/internal/service/revision"
"github.com/apache/answer/internal/service/service_config"
"github.com/apache/answer/internal/service/siteinfo_common"
"github.com/apache/answer/pkg/checker"
"github.com/apache/answer/pkg/dir"
"github.com/apache/answer/pkg/writer"
"github.com/segmentfault/pacman/log"
)
// FileRecordRepo file record repository
type FileRecordRepo interface {
AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)
UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error)
GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) (
fileRecordList []*entity.FileRecord, total int64, err error)
DeleteFileRecord(ctx context.Context, id int) (err error)
}
// FileRecordService file record service
type FileRecordService struct {
fileRecordRepo FileRecordRepo
revisionRepo revision.RevisionRepo
serviceConfig *service_config.ServiceConfig
siteInfoService siteinfo_common.SiteInfoCommonService
}
// NewFileRecordService new file record service
func NewFileRecordService(
fileRecordRepo FileRecordRepo,
revisionRepo revision.RevisionRepo,
serviceConfig *service_config.ServiceConfig,
siteInfoService siteinfo_common.SiteInfoCommonService,
) *FileRecordService {
return &FileRecordService{
fileRecordRepo: fileRecordRepo,
revisionRepo: revisionRepo,
serviceConfig: serviceConfig,
siteInfoService: siteInfoService,
}
}
// AddFileRecord add file record
func (fs *FileRecordService) AddFileRecord(ctx context.Context, userID, filePath, fileURL, source string) {
record := &entity.FileRecord{
UserID: userID,
FilePath: filePath,
FileURL: fileURL,
Source: source,
Status: entity.FileRecordStatusAvailable,
ObjectID: "0",
}
if err := fs.fileRecordRepo.AddFileRecord(ctx, record); err != nil {
log.Errorf("add file record error: %v", err)
}
}
// CleanOrphanUploadFiles clean orphan upload files
func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) {
page, pageSize := 1, 1000
for {
fileRecordList, total, err := fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{
Status: entity.FileRecordStatusAvailable,
})
if err != nil {
log.Errorf("get file record page error: %v", err)
return
}
if len(fileRecordList) == 0 || total == 0 {
break
}
for _, fileRecord := range fileRecordList {
// If this file record created in 48 hours, no need to check
if fileRecord.CreatedAt.AddDate(0, 0, 2).After(time.Now()) {
continue
}
if checker.IsNotZeroString(fileRecord.ObjectID) {
_, exist, err := fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID)
if err != nil {
log.Errorf("get last revision by object id error: %v", err)
continue
}
if exist {
continue
}
} else {
lastRevision, exist, err := fs.revisionRepo.GetLastRevisionByFileURL(ctx, fileRecord.FileURL)
if err != nil {
log.Errorf("get last revision by file url error: %v", err)
continue
}
if exist {
// update the file record object id
fileRecord.ObjectID = lastRevision.ObjectID
if err := fs.fileRecordRepo.UpdateFileRecord(ctx, fileRecord); err != nil {
log.Errorf("update file record object id error: %v", err)
}
continue
}
}
// Delete and move the file record
if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); err != nil {
log.Error(err)
}
}
page++
}
}
func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) {
deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath)
log.Infof("purge deleted files: %s", deletedPath)
err := os.RemoveAll(deletedPath)
if err != nil {
log.Errorf("purge deleted files error: %v", err)
return
}
err = dir.CreateDirIfNotExist(deletedPath)
if err != nil {
log.Errorf("create deleted directory error: %v", err)
}
return
}
func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error {
// Delete the file record
if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err != nil {
return fmt.Errorf("delete file record error: %v", err)
}
// Move the file to the deleted directory
oldFilename := filepath.Base(fileRecord.FilePath)
oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, fileRecord.FilePath)
deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath, oldFilename)
if err := writer.MoveFile(oldFilePath, deletedPath); err != nil {
return fmt.Errorf("move file error: %v", err)
}
log.Debugf("delete and move file: %s", fileRecord.FileURL)
return nil
}
internal/service/object_info/object_info.go
+2 -1
View File
@@ -21,6 +21,7 @@ package object_info
import (
"context"
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/base/reason"
"github.com/apache/answer/internal/schema"
@@ -279,7 +280,7 @@ func (os *ObjService) GetInfo(ctx context.Context, objectID string) (objInfo *sc
ObjectID: tagInfo.ID,
TagID: tagInfo.ID,
ObjectType: objectType,
Title: tagInfo.ParsedText,
Title: tagInfo.SlugName,
Content: tagInfo.ParsedText, // todo trim
}
}
internal/service/provider.go
+2
View File
@@ -36,6 +36,7 @@ import (
"github.com/apache/answer/internal/service/dashboard"
"github.com/apache/answer/internal/service/event_queue"
"github.com/apache/answer/internal/service/export"
"github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/service/follow"
"github.com/apache/answer/internal/service/importer"
"github.com/apache/answer/internal/service/meta"
@@ -126,4 +127,5 @@ var ProviderSetService = wire.NewSet(
badge.NewBadgeAwardService,
badge.NewBadgeGroupService,
importer.NewImporterService,
file_record.NewFileRecordService,
)
internal/service/revision/revision.go
+1
View File
@@ -31,6 +31,7 @@ type RevisionRepo interface {
AddRevision(ctx context.Context, revision *entity.Revision, autoUpdateRevisionID bool) (err error)
GetRevisionByID(ctx context.Context, revisionID string) (revision *entity.Revision, exist bool, err error)
GetLastRevisionByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)
GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error)
GetRevisionList(ctx context.Context, revision *entity.Revision) (revisionList []entity.Revision, err error)
UpdateObjectRevisionId(ctx context.Context, revision *entity.Revision, session *xorm.Session) (err error)
ExistUnreviewedByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error)
internal/service/service_config/service_config.go
+4 -1
View File
@@ -20,5 +20,8 @@
package service_config
type ServiceConfig struct {
UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`
UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"`
CleanUpUploads bool `json:"clean_up_uploads" mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"`
CleanOrphanUploadsPeriodHours int `json:"clean_orphan_uploads_period_hours" mapstructure:"clean_orphan_uploads_period_hours" yaml:"clean_orphan_uploads_period_hours"`
PurgeDeletedFilesPeriodDays int `json:"purge_deleted_files_period_days" mapstructure:"purge_deleted_files_period_days" yaml:"purge_deleted_files_period_days"`
}
internal/service/tag_common/tag_common.go
+8 -1
View File
@@ -349,10 +349,17 @@ func (ts *TagCommonService) AddTag(ctx context.Context, req *schema.AddTagReq) (
}
tagInfoJson, _ := json.Marshal(tagInfo)
revisionDTO.Content = string(tagInfoJson)
_, err = ts.revisionService.AddRevision(ctx, revisionDTO, true)
revisionID, err := ts.revisionService.AddRevision(ctx, revisionDTO, true)
if err != nil {
return nil, err
}
ts.activityQueueService.Send(ctx, &schema.ActivityMsg{
UserID: req.UserID,
ObjectID: tagInfo.ID,
OriginalObjectID: tagInfo.ID,
ActivityTypeKey: constant.ActTagCreated,
RevisionID: revisionID,
})
return &schema.AddTagResp{SlugName: tagInfo.SlugName}, nil
}
internal/service/uploader/upload.go
+35 -17
View File
@@ -31,6 +31,8 @@ import (
"path/filepath"
"strings"
"github.com/apache/answer/internal/service/file_record"
"github.com/apache/answer/internal/base/constant"
"github.com/apache/answer/internal/base/reason"
"github.com/apache/answer/internal/service/service_config"
@@ -53,6 +55,7 @@ var (
constant.PostSubPath,
constant.BrandingSubPath,
constant.FilesPostSubPath,
constant.DeletedSubPath,
}
supportedThumbFileExtMapping = map[string]imaging.Format{
".jpg": imaging.JPEG,
@@ -63,22 +66,26 @@ var (
)
type UploaderService interface {
UploadAvatarFile(ctx *gin.Context) (url string, err error)
UploadPostFile(ctx *gin.Context) (url string, err error)
UploadPostAttachment(ctx *gin.Context) (url string, err error)
UploadBrandingFile(ctx *gin.Context) (url string, err error)
UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error)
UploadPostFile(ctx *gin.Context, userID string) (url string, err error)
UploadPostAttachment(ctx *gin.Context, userID string) (url string, err error)
UploadBrandingFile(ctx *gin.Context, userID string) (url string, err error)
AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error)
}
// uploaderService uploader service
type uploaderService struct {
serviceConfig *service_config.ServiceConfig
siteInfoService siteinfo_common.SiteInfoCommonService
serviceConfig *service_config.ServiceConfig
siteInfoService siteinfo_common.SiteInfoCommonService
fileRecordService *file_record.FileRecordService
}
// NewUploaderService new upload service
func NewUploaderService(serviceConfig *service_config.ServiceConfig,
siteInfoService siteinfo_common.SiteInfoCommonService) UploaderService {
func NewUploaderService(
serviceConfig *service_config.ServiceConfig,
siteInfoService siteinfo_common.SiteInfoCommonService,
fileRecordService *file_record.FileRecordService,
) UploaderService {
for _, subPath := range subPathList {
err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath))
if err != nil {
@@ -86,13 +93,14 @@ func NewUploaderService(serviceConfig *service_config.ServiceConfig,
}
}
return &uploaderService{
serviceConfig: serviceConfig,
siteInfoService: siteInfoService,
serviceConfig: serviceConfig,
siteInfoService: siteInfoService,
fileRecordService: fileRecordService,
}
}
// UploadAvatarFile upload avatar file
func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err error) {
func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar)
if err != nil {
return "", err
@@ -174,7 +182,7 @@ func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, si
return saveFilePath, nil
}
func (us *uploaderService) UploadPostFile(ctx *gin.Context) (
func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserPost)
if err != nil {
@@ -202,10 +210,15 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context) (
fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
avatarFilePath := path.Join(constant.PostSubPath, newFilename)
return us.uploadImageFile(ctx, fileHeader, avatarFilePath)
url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath)
if err != nil {
return "", err
}
us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost))
return url, nil
}
func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (
func (us *uploaderService) UploadPostAttachment(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment)
if err != nil {
@@ -232,11 +245,16 @@ func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) (
fileExt := strings.ToLower(path.Ext(fileHeader.Filename))
newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt)
avatarFilePath := path.Join(constant.FilesPostSubPath, newFilename)
return us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, avatarFilePath)
attachmentFilePath := path.Join(constant.FilesPostSubPath, newFilename)
url, err = us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, attachmentFilePath)
if err != nil {
return "", err
}
us.fileRecordService.AddFileRecord(ctx, userID, attachmentFilePath, url, string(plugin.UserPostAttachment))
return url, nil
}
func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) (
func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) (
url string, err error) {
url, err = us.tryToUploadByPlugin(ctx, plugin.AdminBranding)
if err != nil {
pkg/writer/writer.go
+5
View File
@@ -48,3 +48,8 @@ func WriteFile(filePath, content string) error {
}
return nil
}
// MoveFile move file to new path
func MoveFile(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
}