gitea_hat/services/repository/files/update.go

package files

import (
	"bytes"
	"fmt"
	"path"
	"strings"

	"code.gitea.io/gitea/models"
	git_model "code.gitea.io/gitea/models/git"
	repo_model "code.gitea.io/gitea/models/repo"
	user_model "code.gitea.io/gitea/models/user"
	"code.gitea.io/gitea/modules/charset"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/lfs"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/util"
	asymkey_service "code.gitea.io/gitea/services/asymkey"
	gitea_files_service "code.gitea.io/gitea/services/repository/files"
	"code.gitlink.org.cn/Gitlink/gitea_hat.git/modules/structs"
	"github.com/gobwas/glob"
	stdcharset "golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
)

type FileActionType int

const (
	ActionTypeCreate FileActionType = iota + 1
	ActionTypeUpdate
	ActionTypeDelete
)

var fileActionTypes = map[string]FileActionType{
	"create": ActionTypeCreate,
	"update": ActionTypeUpdate,
	"delete": ActionTypeDelete,
}

func ToFileActionType(name string) FileActionType {
	return fileActionTypes[name]
}

type ExchangeFileOption struct {
	FileChan chan BatchSingleFileOption
	StopChan chan bool
	ErrChan  chan error
}

type BatchSingleFileOption struct {
	Content      string
	TreePath     string
	FromTreePath string
	ActionType   FileActionType
}

type BatchUpdateFileOptions struct {
	Files        []BatchSingleFileOption
	LastCommitID string
	OldBranch    string
	NewBranch    string
	Message      string
	SHA          string
	Author       *gitea_files_service.IdentityOptions
	Commiter     *gitea_files_service.IdentityOptions
	Dates        *gitea_files_service.CommitDateOptions
	Signoff      bool
}

func CreateOrUpdateOrDeleteRepofiles(ctx *context.APIContext, repo *repo_model.Repository, doer *user_model.User, opts *BatchUpdateFileOptions, exchange *ExchangeFileOption) (*structs.BatchFileResponse, error) {

	var protectedPatterns []glob.Glob
	var protectedBranch *git_model.ProtectedBranch
	if opts.OldBranch == "" {
		opts.OldBranch = repo.DefaultBranch
	}

	if opts.NewBranch == "" {
		opts.NewBranch = opts.OldBranch
	}

	gitRepo, closer, err := git.RepositoryFromContextOrOpen(ctx, repo.RepoPath())
	if err != nil {
		return nil, err
	}
	defer closer.Close()

	// oldBranch must exist for this operation
	if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty {
		return nil, err
	}

	if opts.NewBranch != opts.OldBranch {
		existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
		if existingBranch != nil {
			return nil, models.ErrBranchAlreadyExists{
				BranchName: opts.NewBranch,
			}
		}
		if err != nil && !git.IsErrBranchNotExist(err) {
			return nil, err
		}
	} else {
		protectedBranch, err = git_model.GetProtectedBranchBy(ctx, repo.ID, opts.OldBranch)
		if err != nil {
			return nil, err
		}
	}

	message := strings.TrimSpace(opts.Message)
	author, commiter := gitea_files_service.GetAuthorAndCommitterUsers(opts.Author, opts.Commiter, doer)

	t, err := gitea_files_service.NewTemporaryUploadRepository(ctx, repo)
	if err != nil {
		log.Error("%v", err)
	}
	defer t.Close()
	hasOldBranch := true

	if err := t.Clone(opts.OldBranch); err != nil {
		if !git.IsErrBranchNotExist(err) || !repo.IsEmpty {
			return nil, err
		}
		if err := t.Init(); err != nil {
			return nil, err
		}
		hasOldBranch = false
		opts.LastCommitID = ""
	}
	if hasOldBranch {
		if err := t.SetDefaultIndex(); err != nil {
			return nil, err
		}
	}

	// Get the commit of the original branch
	commit, err := t.GetBranchCommit(opts.OldBranch)
	if err != nil {
		return nil, err
	}

	if opts.LastCommitID == "" {
		opts.LastCommitID = commit.ID.String()
	} else {
		lastCommitID, err := gitRepo.ConvertToSHA1(opts.LastCommitID)
		if err != nil {
			return nil, fmt.Errorf("ConvertToSHA1: Invalid last commit ID: %v", err)
		}
		opts.LastCommitID = lastCommitID.String()
	}
	var commitHash string
	var treeNames []string

	for {
		select {
		case file := <-exchange.FileChan:
			for _, pat := range protectedPatterns {
				if pat.Match(strings.ToLower(file.TreePath)) {
					return nil, models.ErrFilePathProtected{
						Path: file.TreePath,
					}
				}
			}
			if protectedBranch != nil {
				isUnprotectedFile := false
				glob := protectedBranch.GetUnprotectedFilePatterns()
				if len(glob) != 0 {
					isUnprotectedFile = protectedBranch.IsUnprotectedFile(glob, file.TreePath)
				}
				if !protectedBranch.CanUserPush(doer.ID) && !isUnprotectedFile {
					return nil, models.ErrUserCannotCommit{
						UserName: doer.LowerName,
					}
				}
				if protectedBranch.RequireSignedCommits {
					_, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), opts.OldBranch)
					if err != nil {
						if !asymkey_service.IsErrWontSign(err) {
							return nil, err
						}
						return nil, models.ErrUserCannotCommit{
							UserName: doer.LowerName,
						}
					}
				}
				patterns := protectedBranch.GetProtectedFilePatterns()
				for _, pat := range patterns {
					if pat.Match(strings.ToLower(file.TreePath)) {
						return nil, models.ErrFilePathProtected{
							Path: file.TreePath,
						}
					}
				}
			}

			optTreePath := file.TreePath
			optFromTreePath := file.FromTreePath
			optActionType := file.ActionType
			optContent := file.Content

			if optTreePath != "" && optFromTreePath == "" {
				optFromTreePath = optTreePath
			}

			treePath := gitea_files_service.CleanUploadFileName(optTreePath)
			if treePath == "" {
				return nil, models.ErrFilenameInvalid{
					Path: optTreePath,
				}
			}

			fromTreePath := gitea_files_service.CleanUploadFileName(optFromTreePath)
			if fromTreePath == "" && optFromTreePath != "" {
				return nil, models.ErrFilenameInvalid{
					Path: optFromTreePath,
				}
			}

			if optActionType == ActionTypeDelete {
				// Get the files in the index
				filesInIndex, err := t.LsFiles(optTreePath)
				if err != nil {
					return nil, fmt.Errorf("DeleteRepoFile: %v", err)
				}
				// Find the file we want to delete in the index
				inFilelist := false
				for _, file := range filesInIndex {
					if file == optTreePath {
						inFilelist = true
						break
					}
				}
				if !inFilelist {
					return nil, models.ErrRepoFileDoesNotExist{
						Path: optTreePath,
					}
				}

				// Get the entry of treePath and check if the SHA given is the same as the file
				entry, err := commit.GetTreeEntryByPath(treePath)
				if err != nil {
					return nil, err
				}
				if opts.SHA != "" {
					// If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error
					if opts.SHA != entry.ID.String() {
						return nil, models.ErrSHADoesNotMatch{
							Path:       treePath,
							GivenSHA:   opts.SHA,
							CurrentSHA: entry.ID.String(),
						}
					}
				} else if opts.LastCommitID != "" {
					// If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw
					// an error, but only if we aren't creating a new branch.
					if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch {
						// CommitIDs don't match, but we don't want to throw a ErrCommitIDDoesNotMatch unless
						// this specific file has been edited since opts.LastCommitID
						if changed, err := commit.FileChangedSinceCommit(treePath, opts.LastCommitID); err != nil {
							return nil, err
						} else if changed {
							return nil, models.ErrCommitIDDoesNotMatch{
								GivenCommitID:   opts.LastCommitID,
								CurrentCommitID: opts.LastCommitID,
							}
						}
						// The file wasn't modified, so we are good to delete it
					}
				} else {
					// When deleting a file, a lastCommitID or SHA needs to be given to make sure other commits haven't been
					// made. We throw an error if one wasn't provided.
					return nil, models.ErrSHAOrCommitIDNotProvided{}
				}

				// Remove the file from the index
				if err := t.RemoveFilesFromIndex(optTreePath); err != nil {
					return nil, err
				}

			} else {
				encoding := "UTF-8"
				bom := false
				executable := false

				if optActionType == ActionTypeUpdate {
					fromEntry, err := commit.GetTreeEntryByPath(fromTreePath)
					if err != nil {
						return nil, err
					}
					if opts.SHA != "" {
						if opts.SHA != fromEntry.ID.String() {
							return nil, models.ErrSHADoesNotMatch{
								Path:       optTreePath,
								GivenSHA:   opts.SHA,
								CurrentSHA: fromEntry.ID.String(),
							}
						}
					} else if opts.LastCommitID != "" {
						if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch {
							if changed, err := commit.FileChangedSinceCommit(treePath, opts.LastCommitID); err != nil {
								return nil, err
							} else if changed {
								return nil, models.ErrCommitIDDoesNotMatch{
									GivenCommitID:   opts.LastCommitID,
									CurrentCommitID: opts.LastCommitID,
								}
							}
						}
					} else {
						return nil, models.ErrSHAOrCommitIDNotProvided{}
					}
					encoding, bom = detectEncodingAndBOM(fromEntry, repo)
					executable = fromEntry.IsExecutable()
				}

				treePathParts := strings.Split(treePath, "/")
				subTreePath := ""
				for index, part := range treePathParts {
					subTreePath = path.Join(subTreePath, part)
					entry, err := commit.GetTreeEntryByPath(subTreePath)
					if err != nil {
						if git.IsErrNotExist(err) {
							break
						}
						return nil, err
					}
					if index < len(treePathParts)-1 {
						if !entry.IsDir() {
							return nil, models.ErrFilePathInvalid{
								Message: fmt.Sprintf("a file exists where you’re trying to create a subdirectory [path: %s]", subTreePath),
								Path:    subTreePath,
								Name:    part,
								Type:    git.EntryModeBlob,
							}
						}
					} else if entry.IsLink() {
						return nil, models.ErrFilePathInvalid{
							Message: fmt.Sprintf("a symbolic link exists where you’re trying to create a subdirectory [path: %s]", subTreePath),
							Path:    subTreePath,
							Name:    part,
							Type:    git.EntryModeSymlink,
						}
					} else if entry.IsDir() {
						return nil, models.ErrFilePathInvalid{
							Message: fmt.Sprintf("a directory exists where you’re trying to create a file [path: %s]", subTreePath),
							Path:    subTreePath,
							Name:    part,
							Type:    git.EntryModeTree,
						}
					} else if fromTreePath != treePath || optActionType == ActionTypeCreate {
						return nil, models.ErrRepoFileAlreadyExists{
							Path: treePath,
						}
					}
				}
				// Get the two paths (might be the same if not moving) from the index if they exist
				filesInIndex, err := t.LsFiles(optTreePath, optFromTreePath)
				if err != nil {
					return nil, fmt.Errorf("UpdateRepoFile: %v", err)
				}

				if optActionType == ActionTypeCreate {
					for _, file := range filesInIndex {
						if file == optTreePath {
							return nil, models.ErrRepoFileAlreadyExists{
								Path: optTreePath,
							}
						}
					}
				}

				// Remove the old path from the tree
				if fromTreePath != treePath && len(filesInIndex) > 0 {
					for _, file := range filesInIndex {
						if file == fromTreePath {
							if err := t.RemoveFilesFromIndex(optFromTreePath); err != nil {
								return nil, err
							}
						}
					}
				}

				content := optContent
				if bom {
					content = string(charset.UTF8BOM) + content
				}
				if encoding != "UTF-8" {
					charsetEncoding, _ := stdcharset.Lookup(encoding)
					if charsetEncoding != nil {
						result, _, err := transform.String(charsetEncoding.NewEncoder(), content)
						if err != nil {
							log.Error("Error re-encoding %s (%s) as %s - will stay as UTF-8: %v", optTreePath, optFromTreePath, encoding, err)
							result = content
						}
						content = result
					} else {
						log.Error("Unknown encoding: %s", encoding)
					}
				}

				optContent = content
				var lfsMetaObject *git_model.LFSMetaObject
				if setting.LFS.StartServer {
					filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
						Attributes: []git.CmdArg{"filter"},
						Filenames:  []string{treePath},
						CachedOnly: true,
					})
					if err != nil {
						return nil, err
					}

					if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" {
						pointer, err := lfs.GeneratePointer(strings.NewReader(optContent))
						if err != nil {
							return nil, err
						}
						lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repo.ID}
						content = pointer.StringContent()
					}
				}

				objectHash, err := t.HashObject(strings.NewReader(content))
				if err != nil {
					return nil, err
				}

				if executable {
					if err := t.AddObjectToIndex("100755", objectHash, treePath); err != nil {
						return nil, err
					}
				} else {
					if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil {
						return nil, err
					}
				}

				if lfsMetaObject != nil {
					lfsMetaObject, err = git_model.NewLFSMetaObject(lfsMetaObject)
					if err != nil {
						return nil, err
					}
					contentStore := lfs.NewContentStore()
					exist, err := contentStore.Exists(lfsMetaObject.Pointer)
					if err != nil {
						return nil, err
					}
					if !exist {
						if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(optContent)); err != nil {
							if _, err2 := git_model.RemoveLFSMetaObjectByOid(repo.ID, lfsMetaObject.Oid); err2 != nil {
								return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
							}
							return nil, err
						}
					}
				}
			}
			opts.Files = append(opts.Files, file)
			treeNames = append(treeNames, file.TreePath)
		case err := <-exchange.ErrChan:
			return nil, err
		case _ = <-exchange.StopChan:
			goto end
		}
	}
end:
	// Now write the tree
	treeHash, err := t.WriteTree()
	if err != nil {
		return nil, err
	}

	// Now commit the tree
	if opts.Dates != nil {
		commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, commiter, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer)
	} else {
		commitHash, err = t.CommitTree(opts.LastCommitID, author, commiter, treeHash, message, opts.Signoff)
	}
	if err != nil {
		return nil, err
	}

	if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
		log.Error("%T %v", err, err)
		return nil, err
	}

	commit, err = t.GetCommit(commitHash)
	if err != nil {
		return nil, err
	}

	file, err := GetBatchFileResponseFromCommit(ctx, repo, commit, opts.NewBranch, treeNames)
	if err != nil {
		return nil, err
	}

	return file, nil
}

func detectEncodingAndBOM(entry *git.TreeEntry, repo *repo_model.Repository) (string, bool) {
	reader, err := entry.Blob().DataAsync()
	if err != nil {
		// return default
		return "UTF-8", false
	}
	defer reader.Close()
	buf := make([]byte, 1024)
	n, err := util.ReadAtMost(reader, buf)
	if err != nil {
		// return default
		return "UTF-8", false
	}
	buf = buf[:n]

	if setting.LFS.StartServer {
		pointer, _ := lfs.ReadPointerFromBuffer(buf)
		if pointer.IsValid() {
			meta, err := git_model.GetLFSMetaObjectByOid(repo.ID, pointer.Oid)
			if err != nil && err != git_model.ErrLFSObjectNotExist {
				// return default
				return "UTF-8", false
			}
			if meta != nil {
				dataRc, err := lfs.ReadMetaObject(pointer)
				if err != nil {
					// return default
					return "UTF-8", false
				}
				defer dataRc.Close()
				buf = make([]byte, 1024)
				n, err = util.ReadAtMost(dataRc, buf)
				if err != nil {
					// return default
					return "UTF-8", false
				}
				buf = buf[:n]
			}
		}
	}

	encoding, err := charset.DetectEncoding(buf)
	if err != nil {
		// just default to utf-8 and no bom
		return "UTF-8", false
	}
	if encoding == "UTF-8" {
		return encoding, bytes.Equal(buf[0:3], charset.UTF8BOM)
	}
	charsetEncoding, _ := stdcharset.Lookup(encoding)
	if charsetEncoding == nil {
		return "UTF-8", false
	}

	result, n, err := transform.String(charsetEncoding.NewDecoder(), string(buf))
	if err != nil {
		// return default
		return "UTF-8", false
	}

	if n > 2 {
		return encoding, bytes.Equal([]byte(result)[0:3], charset.UTF8BOM)
	}

	return encoding, false
}