openinfra
/
openstackid-resources
Code Issues Proposed changes

File Upload - Chunk Support 

new endpoints for uploads

POST api/public/v1/files/upload

Change-Id: If35c616eb243bf5ec66205fb630fe30ce4dad647
Signed-off-by: smarcet <smarcet@gmail.com>
This commit is contained in:
smarcet 2020-09-29 17:05:34 -03:00
parent 0aad82be2f
commit ff52e8047e
13 changed files with 1119 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
app/Http/Controllers/Apis/Protected/Main/OAuth2ChunkedFilesApiController.php Normal file
View File

@ -0,0 +1,57 @@
<?php namespace App\Http\Controllers;
/**
* Copyright 2020 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use Illuminate\Http\JsonResponse;
use Pion\Laravel\ChunkUpload\Exceptions\UploadMissingFileException;
use Pion\Laravel\ChunkUpload\Handler\AbstractHandler;
use Pion\Laravel\ChunkUpload\Receiver\FileReceiver;
/**
* Class OAuth2ChunkedFilesApiController
* @package App\Http\Controllers
*/
class OAuth2ChunkedFilesApiController extends UploadController
{
/**
* Handles the file upload
*
* @param FileReceiver $receiver
*
* @return JsonResponse
*
* @throws UploadMissingFileException
*
*/
public function uploadFile(FileReceiver $receiver)
{
// check if the upload is success, throw exception or return response you need
if ($receiver->isUploaded() === false) {
throw new UploadMissingFileException();
}
// receive the file
$save = $receiver->receive();
// check if the upload has finished (in chunk mode it will send smaller files)
if ($save->isFinished()) {
// save the file and return any response you need
return $this->saveFile($save->getFile());
}
// we are in chunk mode, lets send the current progress
/** @var AbstractHandler $handler */
$handler = $save->handler();
$done = $handler->getPercentageDone();
return response()->json([
"done" => $done
]);
}
}

115
app/Http/Controllers/Apis/Protected/Main/UploadController.php Normal file
View File

@ -0,0 +1,115 @@
<?php namespace App\Http\Controllers;
/**
* Copyright 2020 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use Illuminate\Routing\Controller as BaseController;
use Illuminate\Http\JsonResponse;
use Pion\Laravel\ChunkUpload\Exceptions\UploadFailedException;
use Illuminate\Support\Facades\Storage;
use Illuminate\Http\Request;
use Illuminate\Http\UploadedFile;
use Pion\Laravel\ChunkUpload\Exceptions\UploadMissingFileException;
use Pion\Laravel\ChunkUpload\Handler\AbstractHandler;
use Pion\Laravel\ChunkUpload\Handler\HandlerFactory;
use Pion\Laravel\ChunkUpload\Receiver\FileReceiver;
/**
* Class UploadController
* @package App\Http\Controllers
*/
class UploadController extends BaseController
{
/**
* Handles the file upload
*
* @param Request $request
*
* @return JsonResponse
*
* @throws UploadMissingFileException
* @throws UploadFailedException
*/
public function upload(Request $request) {
// create the file receiver
$receiver = new FileReceiver("file", $request, HandlerFactory::classFromRequest($request));
// check if the upload is success, throw exception or return response you need
if ($receiver->isUploaded() === false) {
throw new UploadMissingFileException();
}
// receive the file
$save = $receiver->receive();
// check if the upload has finished (in chunk mode it will send smaller files)
if ($save->isFinished()) {
// save the file and return any response you need, current example uses `move` function. If you are
// not using move, you need to manually delete the file by unlink($save->getFile()->getPathname())
return $this->saveFile($save->getFile());
}
// we are in chunk mode, lets send the current progress
/** @var AbstractHandler $handler */
$handler = $save->handler();
return response()->json([
"done" => $handler->getPercentageDone(),
'status' => true
]);
}
/**
* Saves the file
*
* @param UploadedFile $file
*
* @return JsonResponse
*/
protected function saveFile(UploadedFile $file)
{
$fileName = $this->createFilename($file);
// Group files by mime type
$mime = str_replace('/', '-', $file->getMimeType());
// Group files by the date (week
$dateFolder = date("Y-m-W");
// Build the file path
$filePath = "upload/{$mime}/{$dateFolder}/";
$disk = Storage::disk('local');
$disk->putFileAs($filePath, $file, $fileName);
unlink($file->getPathname());
return response()->json([
'path' => $filePath,
'name' => $fileName,
'mime_type' => $mime
]);
}
/**
* Create unique filename for uploaded file
* @param UploadedFile $file
* @return string
*/
protected function createFilename(UploadedFile $file)
{
$extension = $file->getClientOriginalExtension();
$filename = str_replace(".".$extension, "", $file->getClientOriginalName()); // Filename without extension
// Add timestamp hash to name of the file
$filename .= "_" . md5(time()) . "." . $extension;
return $filename;
}
}

1
app/Http/Controllers/Apis/Protected/Summit/OAuth2SummitMediaUploadTypeApiController.php
View File

@ -110,6 +110,7 @@ final class OAuth2SummitMediaUploadTypeApiController extends OAuth2ProtectedCont
'name' => 'required|string|max:255',
'description' => 'sometimes|string|max:255',
'is_mandatory' => 'required|boolean',
// in KB
'max_size' => 'required|int|megabyte_aligned',
'private_storage_type' => 'required|string|in:'.implode(",", IStorageTypesConstants::ValidTypes),
'public_storage_type' => 'required|string|in:'.implode(",", IStorageTypesConstants::ValidTypes),

2
app/Http/Middleware/ParseMultipartFormDataInputForNonPostRequests.php
View File

@ -30,7 +30,7 @@ final class ParseMultipartFormDataInputForNonPostRequests
*/
public function handle($request, Closure $next)
{
if ($request->method() == 'POST' OR $request->method() == 'GET') {
if ($request->method() == 'POST' || $request->method() == 'GET') {
return $next($request);
}

5
app/Http/Routes/public.php
View File

@ -28,6 +28,11 @@ Route::group([
]
], function () {
// files
Route::group(['prefix' => 'files'], function () {
Route::post('upload','OAuth2ChunkedFilesApiController@uploadFile' );
});
// members
Route::group(['prefix' => 'members'], function () {
Route::get('', 'OAuth2MembersApiController@getAll');

152
app/Http/Utils/FileUploadInfo.php Normal file
View File

@ -0,0 +1,152 @@
<?php namespace App\Http\Utils;
/**
* Copyright 2020 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use App\Services\FileSystem\FileNameSanitizer;
use Illuminate\Http\Request as LaravelRequest;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use models\exceptions\ValidationException;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Class FileUploadInfo
* @package App\Http\Utils
*/
final class FileUploadInfo
{
/**
* @var int
*/
private $size;
/**
* @var UploadedFile
*/
private $file;
/**
* @var string
*/
private $fileName;
/**
* @var string
*/
private $fileExt;
/**
* FileUploadInfo constructor.
* @param $file
* @param $fileName
* @param $fileExt
* @param $size
*/
private function __construct(
$file,
$fileName,
$fileExt,
$size
)
{
$this->file = $file;
$this->fileName = $fileName;
$this->fileExt = $fileExt;
$this->size = $size;
}
/**
* @param LaravelRequest $request
* @param array $payload
* @return FileUploadInfo|null
* @throws ValidationException
*/
public static function build(LaravelRequest $request, array $payload):?FileUploadInfo {
$file = null;
$fileName = null;
$fileExt = null;
$size = 0;
if($request->hasFile('file')) {
Log::debug(sprintf("FileUploadInfo::build build file is present on request ( MULTIFORM )"));
$file = $request->file('file');
// get in bytes should be converted to KB
$size = $file->getSize();
if($size == 0)
throw new ValidationException("File size is zero.");
$size = $size/1024; // convert bytes to KB
$fileName = $file->getClientOriginalName();
$fileExt = pathinfo($fileName, PATHINFO_EXTENSION);
}
if(is_null($file) && isset($payload['filepath'])){
Log::debug(sprintf("FileUploadInfo::build build file is present on as local storage (%s)", $payload['filepath']));
$disk = Storage::disk('local');
if(!$disk->exists($payload['filepath']))
throw new ValidationException(sprintf("file provide on filepath %s does not exists on local storage.", $payload['filepath']));
// get in bytes should be converted to KB
$size = $disk->size($payload['filepath']);
if($size == 0)
throw new ValidationException("File size is zero.");
$size = $size/1024; // convert bytes to KB
$fileName = pathinfo($payload['filepath'],PATHINFO_BASENAME);
$fileExt = pathinfo($fileName, PATHINFO_EXTENSION);
$file = new UploadedFile($disk->path($payload['filepath']), $fileName);
}
if(is_null($file)) return null;
$fileName = FileNameSanitizer::sanitize($fileName);
return new self($file, $fileName, $fileExt, $size);
}
/**
* @return int
*/
public function getSize(): ?int
{
return $this->size;
}
/**
* @return UploadedFile
*/
public function getFile(): ?UploadedFile
{
return $this->file;
}
/**
* @return string
*/
public function getFileName(): ?string
{
return $this->fileName;
}
/**
* @return string
*/
public function getFileExt(): ?string
{
return $this->fileExt;
}
public function delete(){
if(is_null($this->file)) return;
$realPath = $this->file->getRealPath();
Log::debug(sprintf("FileUploadInfo::delete deleting file %s", $realPath));
unlink($realPath);
}
}

2
app/Models/Foundation/Summit/Factories/SummitMediaUploadTypeFactory.php
View File

@ -39,7 +39,7 @@ final class SummitMediaUploadTypeFactory
if(isset($data['description']))
$type->setDescription(trim($data['description']));
if(isset($data['max_size']))
if(isset($data['max_size'])) // in KB
$type->setMaxSize(intval($data['max_size']));
if(isset($data['private_storage_type']))

13
app/Models/Foundation/Summit/MediaUploads/SummitMediaUploadType.php
View File

@ -55,6 +55,7 @@ class SummitMediaUploadType extends SilverstripeBaseModel
/**
* @ORM\Column(name="MaxSize", type="integer")
* @var int
* in KB
*/
private $max_size;
@ -173,6 +174,14 @@ class SummitMediaUploadType extends SilverstripeBaseModel
return $this->max_size;
}
/**
* @return int
*/
public function getMaxSizeMB(): int
{
return $this->max_size/1024;
}
/**
* @param int $max_size
*/
@ -268,6 +277,10 @@ class SummitMediaUploadType extends SilverstripeBaseModel
return in_array(strtoupper($ext), explode('|', $this->type->getAllowedExtensions()));
}
public function getValidExtensions(){
return $this->type->getAllowedExtensions();
}
public function hasStorageSet():bool {
return ($this->private_storage_type != IStorageTypesConstants::None || $this->public_storage_type != IStorageTypesConstants::None);
}

110
app/Services/Model/Imp/PresentationService.php
View File

@ -13,6 +13,7 @@
**/
use App\Events\PresentationMaterialDeleted;
use App\Events\PresentationMaterialUpdated;
use App\Http\Utils\FileUploadInfo;
use App\Http\Utils\IFileUploader;
use App\Jobs\Emails\PresentationSubmissions\PresentationCreatorNotificationEmail;
use App\Jobs\Emails\PresentationSubmissions\PresentationSpeakerNotificationEmail;
@ -29,6 +30,7 @@ use App\Models\Foundation\Summit\Events\Presentations\TrackQuestions\TrackAnswer
use Illuminate\Support\Facades\Config;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use models\exceptions\EntityNotFoundException;
use models\exceptions\ValidationException;
use models\main\IFolderRepository;
@ -47,6 +49,8 @@ use libs\utils\ITransactionService;
use models\summit\Summit;
use Illuminate\Http\Request as LaravelRequest;
use App\Services\Model\IFolderService;
use Symfony\Component\HttpFoundation\File\UploadedFile;
/**
* Class PresentationService
* @package services\model
@ -997,78 +1001,67 @@ final class PresentationService
) {
Log::debug(sprintf("PresentationService::addMediaUploadTo summit %s presentation %s", $summit->getId(), $presentation_id));
$presentation = $this->presentation_repository->getById($presentation_id);
if (is_null($presentation) || !$presentation instanceof Presentation)
throw new EntityNotFoundException('Presentation not found.');
$hasFile = $request->hasFile('file');
if(!$hasFile){
throw new ValidationException("You must provide a file.");
}
$media_upload_type_id = intval($payload['media_upload_type_id']);
$media_upload_type = $summit->getMediaUploadTypeById($media_upload_type_id);
$mediaUploadType = $summit->getMediaUploadTypeById($media_upload_type_id);
if(is_null($media_upload_type))
if(is_null($mediaUploadType))
throw new EntityNotFoundException(sprintf("Media Upload Type %s not found.", $media_upload_type_id));
if(!$media_upload_type->isPresentationTypeAllowed($presentation->getType())){
if(!$mediaUploadType->isPresentationTypeAllowed($presentation->getType())){
throw new ValidationException(sprintf("Presentation Type %s is not allowed on Media Upload %s", $presentation->getTypeId(), $media_upload_type_id));
}
$file = $request->file('file');
// get in bytes should be converted to KB
$size = $file->getSize();
if($size == 0)
throw new ValidationException("File size is zero.");
$size = $size/1024;
$fileName = $file->getClientOriginalName();
$fileExt = pathinfo($fileName, PATHINFO_EXTENSION);
// normalize fileName
$fileName = FileNameSanitizer::sanitize($fileName);
if($media_upload_type->getMaxSize() < $size){
throw new ValidationException(sprintf("Max Size is %s KB.", $media_upload_type->getMaxSize()));
}
if(!$media_upload_type->isValidExtension($fileExt)){
throw new ValidationException(sprintf("File Extension %s is not valid", $fileExt));
}
if($presentation->hasMediaUploadByType($media_upload_type)){
if($presentation->hasMediaUploadByType($mediaUploadType)){
throw new ValidationException
(
sprintf
(
"Presentation %s already has a media upload for that type %s.",
$presentation_id, $media_upload_type->getName()
$presentation_id, $mediaUploadType->getName()
)
);
}
$fileInfo = FileUploadInfo::build($request, $payload);
if(is_null($fileInfo)){
throw new ValidationException("You must provide a file.");
}
if($mediaUploadType->getMaxSize() < $fileInfo->getSize()){
throw new ValidationException(sprintf("Max Size is %s MB (%s).", $mediaUploadType->getMaxSizeMB(), $fileInfo->getSize()/1024));
}
if(!$mediaUploadType->isValidExtension($fileInfo->getFileExt())){
throw new ValidationException(sprintf("File Extension %s is not valid (%s).", $fileInfo->getFileExt(), $mediaUploadType->getValidExtensions()));
}
$mediaUpload = PresentationMediaUploadFactory::build(array_merge(
$payload,
[
'media_upload_type' => $media_upload_type,
'media_upload_type' => $mediaUploadType,
'presentation' => $presentation
]
));
$strategy = FileUploadStrategyFactory::build($media_upload_type->getPrivateStorageType());
$strategy = FileUploadStrategyFactory::build($mediaUploadType->getPrivateStorageType());
if(!is_null($strategy)){
$strategy->save($file, $mediaUpload->getPath(IStorageTypesConstants::PrivateType), $fileName);
$strategy->save($fileInfo->getFile(), $mediaUpload->getPath(IStorageTypesConstants::PrivateType), $fileInfo->getFileName());
}
$strategy = FileUploadStrategyFactory::build($media_upload_type->getPublicStorageType());
$strategy = FileUploadStrategyFactory::build($mediaUploadType->getPublicStorageType());
if(!is_null($strategy)){
$strategy->save($file, $mediaUpload->getPath(IStorageTypesConstants::PublicType), $fileName);
$strategy->save($fileInfo->getFile(), $mediaUpload->getPath(IStorageTypesConstants::PublicType), $fileInfo->getFileName());
}
$mediaUpload->setFilename($fileName);
$mediaUpload->setFilename($fileInfo->getFileName());
$presentation->addMediaUpload($mediaUpload);
if(!$presentation->isCompleted()){
@ -1085,6 +1078,8 @@ final class PresentationService
$presentation->setProgress(Presentation::PHASE_UPLOADS);
}
}
Log::debug(sprintf("PresentationService::addMediaUploadTo presentation %s deleting original file %s", $presentation_id, $fileInfo->getFileName()));
$fileInfo->delete();
return $mediaUpload;
});
@ -1116,6 +1111,8 @@ final class PresentationService
$payload
) {
Log::debug(sprintf("PresentationService::updateMediaUploadFrom summit %s presentation %s", $summit->getId(), $presentation_id));
$presentation = $this->presentation_repository->getById($presentation_id);
if (is_null($presentation) || !$presentation instanceof Presentation)
@ -1126,42 +1123,41 @@ final class PresentationService
if (is_null($mediaUpload))
throw new EntityNotFoundException('Presentation Media Upload not found.');
$hasFile = $request->hasFile('file');
if($hasFile) {
$file = $request->file('file');
// get in bytes should be converted to KB
$size = $file->getSize();
if ($size == 0)
throw new ValidationException("File size is zero.");
$size = $size / 1024;
$fileName = $file->getClientOriginalName();
$fileExt = pathinfo($fileName, PATHINFO_EXTENSION);
// normalize fileName
$fileName = FileNameSanitizer::sanitize($fileName);
$fileInfo = FileUploadInfo::build($request, $payload);
if(!is_null($fileInfo)) {
// process file
$mediaUploadType = $mediaUpload->getMediaUploadType();
if (is_null($mediaUploadType))
throw new ValidationException("Media Upload Type is not set.");
if ($mediaUploadType->getMaxSize() < $size) {
throw new ValidationException(sprintf("Max Size is %s KB.", $mediaUploadType->getMaxSize()));
$fileInfo = FileUploadInfo::build($request, $payload);
if(is_null($fileInfo)){
throw new ValidationException("You must provide a file.");
}
if (!$mediaUploadType->isValidExtension($fileExt)) {
throw new ValidationException(sprintf("File Extension %s is not valid", $fileExt));
if($mediaUploadType->getMaxSize() < $fileInfo->getSize()){
throw new ValidationException(sprintf("Max Size is %s MB (%s).", $mediaUploadType->getMaxSizeMB(), $fileInfo->getSize()/1024));
}
if(!$mediaUploadType->isValidExtension($fileInfo->getFileExt())){
throw new ValidationException(sprintf("File Extension %s is not valid (%s).", $fileInfo->getFileExt(), $mediaUploadType->getValidExtensions()));
}
$strategy = FileUploadStrategyFactory::build($mediaUploadType->getPrivateStorageType());
if (!is_null($strategy)) {
$strategy->save($file, $mediaUpload->getPath(IStorageTypesConstants::PrivateType), $fileName);
$strategy->save($fileInfo->getFile(), $mediaUpload->getPath(IStorageTypesConstants::PrivateType), $fileInfo->getFileName());
}
$strategy = FileUploadStrategyFactory::build($mediaUploadType->getPublicStorageType());
if (!is_null($strategy)) {
$strategy->save($file, $mediaUpload->getPath(IStorageTypesConstants::PublicType), $fileName);
$strategy->save($fileInfo->getFile(), $mediaUpload->getPath(IStorageTypesConstants::PublicType), $fileInfo->getFileName());
}
$payload['file_name'] = $fileName;
$payload['file_name'] = $fileInfo->getFileName();
$fileInfo->delete();
}
return PresentationMediaUploadFactory::populate($mediaUpload, $payload);
@ -1179,6 +1175,8 @@ final class PresentationService
$presentation_id,
$media_upload_id
) {
Log::debug(sprintf("PresentationService::deleteMediaUpload summit %s presentation %s", $summit->getId(), $presentation_id));
$presentation = $this->presentation_repository->getById($presentation_id);
if (is_null($presentation) || !$presentation instanceof Presentation)

1
composer.json
View File

@ -38,6 +38,7 @@
"muxinc/mux-php": "^0.5.0",
"php-amqplib/php-amqplib": "^2.11",
"php-opencloud/openstack": "dev-master",
"pion/laravel-chunk-upload": "^1.4",
"predis/predis": "1.0.*",
"s-ichikawa/laravel-sendgrid-driver": "^2.0",
"simplesoftwareio/simple-qrcode": "^2.0",

1034
composer.lock generated
View File

File diff suppressed because it is too large Load Diff

43
config/chunk-upload.php Normal file
View File

@ -0,0 +1,43 @@
<?php
return [
/*
* The storage config
*/
'storage' => [
/*
* Returns the folder name of the chunks. The location is in storage/app/{folder_name}
*/
'chunks' => 'chunks',
'disk' => 'local',
],
'clear' => [
/*
* How old chunks we should delete
*/
'timestamp' => '-3 HOURS',
'schedule' => [
'enabled' => true,
'cron' => '25 * * * *', // run every hour on the 25th minute
],
],
'chunk' => [
// setup for the chunk naming setup to ensure same name upload at same time
'name' => [
'use' => [
'session' => env('CHUNK_UPLOAD_USE_SESSION_ID', true), // should the chunk name use the session id? The uploader must send cookie!,
'browser' => env('CHUNK_UPLOAD_USE_BROWSER_ID', false), // instead of session we can use the ip and browser?
],
],
],
'handlers' => [
// A list of handlers/providers that will be appended to existing list of handlers
'custom' => [
],
// Overrides the list of handlers - use only what you really want
'override' => [
// \Pion\Laravel\ChunkUpload\Handler\DropZoneUploadHandler::class
],
],
];

7
tests/PresentationMediaUploadsTests.php
View File

@ -81,11 +81,12 @@ class PresentationMediaUploadsTests
$headers = [
"HTTP_Authorization" => " Bearer " . $this->access_token,
"CONTENT_TYPE" => "multipart/form-data; boundary=----WebKitFormBoundaryBkSYnzBIiFtZu4pb"
// "CONTENT_TYPE" => "multipart/form-data; boundary=----WebKitFormBoundaryBkSYnzBIiFtZu4pb"
];
$payload = [
'media_upload_type_id' => self::$media_upload_type->getId()
'media_upload_type_id' => self::$media_upload_type->getId(),
'filepath' => 'upload/video-mp4/2020-10-41/OpenDev 2020- Hardware Automation_ab8dbdb02b52fea11b7e3e5e80c63086.mp4'
];
$response = $this->action
@ -96,7 +97,7 @@ class PresentationMediaUploadsTests
$payload,
[],
[
'file' => UploadedFile::fake()->image('slide.png')
//'file' => UploadedFile::fake()->image('slide.png')
],
$headers,
json_encode($payload)