mediaapi/writers/download: Rework remote file download synchronisation

Avoid locking around db requests by only locking around active requests
and always creating an active request if there is none. A nice side
effect of this is that if many parallel requests for remote media come
in (a common case) then only one database query is made for the duration
of the query.
This commit is contained in:
Robert Swain 2017-06-01 12:32:15 +02:00
parent a3b1c7535a
commit 4457ebddca
2 changed files with 69 additions and 98 deletions

View file

@ -18,6 +18,7 @@ import (
"sync" "sync"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
) )
// FileSizeBytes is a file size in bytes // FileSizeBytes is a file size in bytes
@ -63,8 +64,10 @@ type MediaMetadata struct {
type RemoteRequestResult struct { type RemoteRequestResult struct {
// Condition used for the requester to signal the result to all other routines waiting on this condition // Condition used for the requester to signal the result to all other routines waiting on this condition
Cond *sync.Cond Cond *sync.Cond
// Resulting HTTP status code from the request // MediaMetadata of the requested file to avoid querying the database for every waiting routine
Result int MediaMetadata *MediaMetadata
// An error in util.JSONResponse form. nil in case of no error.
ErrorResponse *util.JSONResponse
} }
// ActiveRemoteRequests is a lockable map of media URIs requested from remote homeservers // ActiveRemoteRequests is a lockable map of media URIs requested from remote homeservers

View file

@ -140,10 +140,14 @@ func (r *downloadRequest) doDownload(w http.ResponseWriter, cfg *config.MediaAPI
} }
} }
// If we do not have a record and the origin is remote, we need to fetch it and respond with that file // If we do not have a record and the origin is remote, we need to fetch it and respond with that file
return r.respondFromRemoteFile(w, cfg, db, activeRemoteRequests) resErr := r.getRemoteFile(cfg, db, activeRemoteRequests)
if resErr != nil {
return resErr
}
} else {
// If we have a record, we can respond from the local file
r.MediaMetadata = mediaMetadata
} }
// If we have a record, we can respond from the local file
r.MediaMetadata = mediaMetadata
return r.respondFromLocalFile(w, cfg.AbsBasePath) return r.respondFromLocalFile(w, cfg.AbsBasePath)
} }
@ -207,137 +211,103 @@ func (r *downloadRequest) respondFromLocalFile(w http.ResponseWriter, absBasePat
return nil return nil
} }
// respondFromRemoteFile fetches the remote file, caches it locally and responds from that local file // getRemoteFile fetches the remote file and caches it locally
// A hash map of active remote requests to a struct containing a sync.Cond is used to only download remote files once, // A hash map of active remote requests to a struct containing a sync.Cond is used to only download remote files once,
// regardless of how many download requests are received. // regardless of how many download requests are received.
// Note: The named errorResponse return variable is used in a deferred broadcast of the metadata and error response to waiting goroutines.
// Returns a util.JSONResponse error in case of error // Returns a util.JSONResponse error in case of error
func (r *downloadRequest) respondFromRemoteFile(w http.ResponseWriter, cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) *util.JSONResponse { func (r *downloadRequest) getRemoteFile(cfg *config.MediaAPI, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) (errorResponse *util.JSONResponse) {
// Note: getMediaMetadataForRemoteFile uses mutexes and conditions from activeRemoteRequests // Note: getMediaMetadataFromActiveRequest uses mutexes and conditions from activeRemoteRequests
mediaMetadata, resErr := r.getMediaMetadataForRemoteFile(db, activeRemoteRequests) mediaMetadata, resErr := r.getMediaMetadataFromActiveRequest(activeRemoteRequests)
if resErr != nil { if resErr != nil {
return resErr return resErr
} else if mediaMetadata != nil { } else if mediaMetadata != nil {
// If we have a record, we can respond from the local file // If we got metadata from an active request, we can respond from the local file
r.MediaMetadata = mediaMetadata r.MediaMetadata = mediaMetadata
} else { } else {
// If we do not have a record, we need to fetch the remote file first and then respond from the local file // Note: This is an active request that MUST broadcastMediaMetadata to wake up waiting goroutines!
// Note: getRemoteFile uses mutexes and conditions from activeRemoteRequests // Note: errorResponse is the named return variable
if resErr := r.getRemoteFile(cfg.AbsBasePath, cfg.MaxFileSizeBytes, db, activeRemoteRequests); resErr != nil { // Note: broadcastMediaMetadata uses mutexes and conditions from activeRemoteRequests
return resErr defer r.broadcastMediaMetadata(activeRemoteRequests, errorResponse)
}
}
return r.respondFromLocalFile(w, cfg.AbsBasePath)
}
func (r *downloadRequest) getMediaMetadataForRemoteFile(db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) (*types.MediaMetadata, *util.JSONResponse) {
activeRemoteRequests.Lock()
defer activeRemoteRequests.Unlock()
// check if we have a record of the media in our database
mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
if err != nil {
r.Logger.WithError(err).Error("Error querying the database.")
resErr := jsonerror.InternalServerError()
return nil, &resErr
}
if mediaMetadata != nil {
// If we have a record, we can respond from the local file
return mediaMetadata, nil
}
// No record was found
// Check if there is an active remote request for the file
mxcURL := "mxc://" + string(r.MediaMetadata.Origin) + "/" + string(r.MediaMetadata.MediaID)
if activeRemoteRequestResult, ok := activeRemoteRequests.MXCToResult[mxcURL]; ok {
r.Logger.Info("Waiting for another goroutine to fetch the remote file.")
// NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this.
activeRemoteRequestResult.Cond.Wait()
// check if we have a record of the media in our database // check if we have a record of the media in our database
mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin) mediaMetadata, err := db.GetMediaMetadata(r.MediaMetadata.MediaID, r.MediaMetadata.Origin)
if err != nil { if err != nil {
r.Logger.WithError(err).Error("Error querying the database.") r.Logger.WithError(err).Error("Error querying the database.")
resErr := jsonerror.InternalServerError() resErr := jsonerror.InternalServerError()
return nil, &resErr return &resErr
} }
if mediaMetadata != nil { if mediaMetadata == nil {
// If we do not have a record, we need to fetch the remote file first and then respond from the local file
resErr := r.fetchRemoteFileAndStoreMetadata(cfg.AbsBasePath, cfg.MaxFileSizeBytes, db)
if resErr != nil {
return resErr
}
} else {
// If we have a record, we can respond from the local file // If we have a record, we can respond from the local file
return mediaMetadata, nil r.MediaMetadata = mediaMetadata
}
}
return
}
func (r *downloadRequest) getMediaMetadataFromActiveRequest(activeRemoteRequests *types.ActiveRemoteRequests) (*types.MediaMetadata, *util.JSONResponse) {
// Check if there is an active remote request for the file
mxcURL := "mxc://" + string(r.MediaMetadata.Origin) + "/" + string(r.MediaMetadata.MediaID)
activeRemoteRequests.Lock()
defer activeRemoteRequests.Unlock()
if activeRemoteRequestResult, ok := activeRemoteRequests.MXCToResult[mxcURL]; ok {
r.Logger.Info("Waiting for another goroutine to fetch the remote file.")
// NOTE: Wait unlocks and locks again internally. There is still a deferred Unlock() that will unlock this.
activeRemoteRequestResult.Cond.Wait()
if activeRemoteRequestResult.ErrorResponse != nil {
return nil, activeRemoteRequestResult.ErrorResponse
} }
// Note: if the result was 200, we shouldn't get here if activeRemoteRequestResult.MediaMetadata == nil {
switch activeRemoteRequestResult.Result {
case 404:
return nil, &util.JSONResponse{ return nil, &util.JSONResponse{
Code: 404, Code: 404,
JSON: jsonerror.NotFound("File not found."), JSON: jsonerror.NotFound("File not found."),
} }
case 500:
r.Logger.Error("Other goroutine failed to fetch the remote file.")
resErr := jsonerror.InternalServerError()
return nil, &resErr
default:
r.Logger.Error("Other goroutine failed to fetch the remote file.")
return nil, &util.JSONResponse{
Code: activeRemoteRequestResult.Result,
JSON: jsonerror.Unknown("Failed to fetch file from remote server."),
}
} }
return activeRemoteRequestResult.MediaMetadata, nil
} }
// No active remote request so create one // No active remote request so create one
activeRemoteRequests.MXCToResult[mxcURL] = &types.RemoteRequestResult{ activeRemoteRequests.MXCToResult[mxcURL] = &types.RemoteRequestResult{
Cond: &sync.Cond{L: activeRemoteRequests}, Cond: &sync.Cond{L: activeRemoteRequests},
} }
return nil, nil return nil, nil
} }
// getRemoteFile fetches the file from the remote server and stores its metadata in the database // broadcastMediaMetadata broadcasts the media metadata and error response to waiting goroutines
// Only the owner of the activeRemoteRequestResult for this origin and media ID should call this function. // Only the owner of the activeRemoteRequestResult for this origin and media ID should call this function.
func (r *downloadRequest) getRemoteFile(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database, activeRemoteRequests *types.ActiveRemoteRequests) *util.JSONResponse { func (r *downloadRequest) broadcastMediaMetadata(activeRemoteRequests *types.ActiveRemoteRequests, errorResponse *util.JSONResponse) {
// Wake up other goroutines after this function returns. activeRemoteRequests.Lock()
isError := true defer activeRemoteRequests.Unlock()
var result int mxcURL := "mxc://" + string(r.MediaMetadata.Origin) + "/" + string(r.MediaMetadata.MediaID)
defer func() { if activeRemoteRequestResult, ok := activeRemoteRequests.MXCToResult[mxcURL]; ok {
if isError { r.Logger.Info("Signalling other goroutines waiting for this goroutine to fetch the file.")
// If an error happens, the lock MUST NOT have been taken, isError MUST be true and so the lock is taken here. activeRemoteRequestResult.MediaMetadata = r.MediaMetadata
activeRemoteRequests.Lock() activeRemoteRequestResult.ErrorResponse = errorResponse
} activeRemoteRequestResult.Cond.Broadcast()
defer activeRemoteRequests.Unlock() }
mxcURL := "mxc://" + string(r.MediaMetadata.Origin) + "/" + string(r.MediaMetadata.MediaID) delete(activeRemoteRequests.MXCToResult, mxcURL)
if activeRemoteRequestResult, ok := activeRemoteRequests.MXCToResult[mxcURL]; ok { }
r.Logger.Info("Signalling other goroutines waiting for this goroutine to fetch the file.")
if result == 0 {
r.Logger.Error("Invalid result, treating as InternalServerError")
result = 500
}
activeRemoteRequestResult.Result = result
activeRemoteRequestResult.Cond.Broadcast()
}
delete(activeRemoteRequests.MXCToResult, mxcURL)
}()
// fetchRemoteFileAndStoreMetadata fetches the file from the remote server and stores its metadata in the database
func (r *downloadRequest) fetchRemoteFileAndStoreMetadata(absBasePath types.Path, maxFileSizeBytes types.FileSizeBytes, db *storage.Database) *util.JSONResponse {
finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes) finalPath, duplicate, resErr := r.fetchRemoteFile(absBasePath, maxFileSizeBytes)
if resErr != nil { if resErr != nil {
result = resErr.Code
return resErr return resErr
} }
// NOTE: Writing the metadata to the media repository database and removing the mxcURL from activeRemoteRequests needs to be atomic.
// If it were not atomic, a new request for the same file could come in in routine A and check the database before the INSERT.
// Routine B which was fetching could then have its INSERT complete and remove the mxcURL from the activeRemoteRequests.
// If routine A then checked the activeRemoteRequests it would think it needed to fetch the file when it's already in the database.
// The locking below mitigates this situation.
// NOTE: The following two lines MUST remain together!
// isError == true causes the lock to be taken in a deferred function!
activeRemoteRequests.Lock()
isError = false
r.Logger.WithFields(log.Fields{ r.Logger.WithFields(log.Fields{
"Base64Hash": r.MediaMetadata.Base64Hash, "Base64Hash": r.MediaMetadata.Base64Hash,
"UploadName": r.MediaMetadata.UploadName, "UploadName": r.MediaMetadata.UploadName,
@ -357,7 +327,6 @@ func (r *downloadRequest) getRemoteFile(absBasePath types.Path, maxFileSizeBytes
// NOTE: It should really not be possible to fail the uniqueness test here so // NOTE: It should really not be possible to fail the uniqueness test here so
// there is no need to handle that separately // there is no need to handle that separately
resErr := jsonerror.InternalServerError() resErr := jsonerror.InternalServerError()
result = resErr.Code
return &resErr return &resErr
} }
@ -370,7 +339,6 @@ func (r *downloadRequest) getRemoteFile(absBasePath types.Path, maxFileSizeBytes
"Content-Type": r.MediaMetadata.ContentType, "Content-Type": r.MediaMetadata.ContentType,
}).Infof("Remote file cached") }).Infof("Remote file cached")
result = 200
return nil return nil
} }