Skip to content

Commit

Permalink
Merge pull request #3937 from mind84/upload_webdav
Browse files Browse the repository at this point in the history 
[Feature] Use remote WebDAV server to store and retreive files
  • Loading branch information
@mdouchin
mdouchin authored Nov 22, 2023
2 parents dfafd0a + 857c4f4 commit 6cc1ba5
Show file tree
Hide file tree
Showing 34 changed files with 3,314 additions and 77 deletions.
5 changes: 5 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ tests/qgis-projects/*
# Zip file coming with QGIS 3.28
tests/qgis-projects/*/*_attachments.zip
!tests/qgis-projects/tests
!tests/qgis-projects/webdav
tests/qgis-projects/webdav/test/*
!tests/qgis-projects/webdav/test/logo.png
!tests/qgis-projects/webdav/test/test_upload.conf
!tests/qgis-projects/webdav/test/test_upload.txt
!tests/qgis-projects/ProJets 1982*!
tests/qgis-projects/tests/media
tests/qgis-server-plugins/*
Expand Down
15 changes: 14 additions & 1 deletion assets/src/legacy/attributeTable.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1459,7 +1459,6 @@ var lizAttributeTable = function() {
function createDatatableColumns(aName, atFeatures, hiddenFields, cAliases, cTypes, allColumnsKeyValues){
const columns = [];
let firstDisplayedColIndex = 0;

// Column with selected status
columns.push( {"data": "lizSelected", "width": "25px", "searchable": false, "sortable": true, "visible": false} );
firstDisplayedColIndex+=1;
Expand Down Expand Up @@ -1511,17 +1510,31 @@ var lizAttributeTable = function() {
}
} else {
// Check if we need to replace url or media by link
let davConf = lizUrls.webDavUrl && lizUrls?.resourceUrlReplacement?.webdav && config.layers[aName]?.webDavFields && Array.isArray(config.layers[aName].webDavFields) && config.layers[aName].webDavFields.includes(columnName);
colConf['render'] = function (data, type, row, meta) {
// Replace media and URL with links
if (!data || !(typeof data === 'string'))
return data;
if (davConf) {
// replace the root of the url
if(data.startsWith(lizUrls.webDavUrl)){
data = data.replace(lizUrls.webDavUrl, lizUrls.resourceUrlReplacement.webdav)
}
}

if (data.substring(0, 6) == 'media/' || data.substring(0, 7) == '/media/' || data.substring(0, 9) == '../media/') {
var rdata = data;
var colMeta = meta.settings.aoColumns[meta.col];
if (data.substring(0, 7) == '/media/')
rdata = data.slice(1);
return '<a href="' + mediaLinkPrefix + '&path=' + rdata + '" target="_blank">' + colMeta.title + '</a>';
}
else if (davConf && data.substring(0, 4) == lizUrls.resourceUrlReplacement.webdav) {
var rdata = data;
var colMeta = meta.settings.aoColumns[meta.col];
return '<a href="' + mediaLinkPrefix + '&path=' + rdata + '" target="_blank">' + colMeta.title + '</a>';

}
else if (data.substring(0, 4) == 'http' || data.substring(0, 3) == 'www') {
var rdata = data;
if (data.substring(0, 3) == 'www')
Expand Down
16 changes: 16 additions & 0 deletions lizmap/modules/lizmap/classes/qgisVectorLayer.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ class qgisVectorLayer extends qgisMapLayer

protected $wfsFields = array();

protected $webDavFields = array();

protected $webDavBaseUris = array();

/**
* @var null|object connection parameters
*/
Expand Down Expand Up @@ -73,6 +77,8 @@ public function __construct($project, $propLayer)
$this->defaultValues = $propLayer['defaults'];
$this->constraints = $propLayer['constraints'];
$this->wfsFields = $propLayer['wfsFields'];
$this->webDavFields = $propLayer['webDavFields'];
$this->webDavBaseUris = $propLayer['webDavBaseUris'];
}

/**
Expand Down Expand Up @@ -173,6 +179,16 @@ public function getWfsFields()
return $this->wfsFields;
}

public function getWebDavFieldConfiguration()
{
$davConf = array();
foreach ($this->webDavFields as $index => $webDavField) {
$davConf[$webDavField] = $this->webDavBaseUris[$index];
}

return $davConf;
}

/**
* @return object
*/
Expand Down
135 changes: 131 additions & 4 deletions lizmap/modules/lizmap/lib/Form/QgisForm.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
namespace Lizmap\Form;

use Lizmap\App;
use Lizmap\Request\RemoteStorageRequest;

class QgisForm implements QgisFormControlsInterface
{
Expand Down Expand Up @@ -565,9 +566,16 @@ public function check($feature = null)

if ($form->getControl($fieldName) instanceof \jFormsControlUpload2) {
list($targetPath, $targetFullPath) = $this->getStoragePathForControl($fieldName);
// if the target path to store the file is not valid: error
if ($targetFullPath == '' || !is_dir($targetFullPath) || !is_writable($targetFullPath)) {
$form->setErrorOn($fieldName, \jLocale::get('view~edition.message.error.upload.layer', array($dtParams->tablename)));
if ($this->getQgisControl($fieldName)->isWebDAV) {
if (!RemoteStorageRequest::checkWebDAVStorageConnection()) {
$check = false;
$form->setErrorOn($fieldName, 'WEBDAV storage unavailable');
}
} else {
// if the target path to store the file is not valid: error
if ($targetFullPath == '' || !is_dir($targetFullPath) || !is_writable($targetFullPath)) {
$form->setErrorOn($fieldName, \jLocale::get('view~edition.message.error.upload.layer', array($dtParams->tablename)));
}
}
}
}
Expand Down Expand Up @@ -806,7 +814,20 @@ public function saveToDb($feature = null, $modifiedControls = array())
}
// Control is an upload control
if ($jCtrl instanceof \jFormsControlUpload2 || $jCtrl instanceof \jFormsControlUpload) {
$values[$ref] = $this->processUploadedFile($form, $ref);
if (array_key_exists($ref, $this->formControls) && $this->formControls[$ref]->isWebDAV && isset($this->formControls[$ref]->webDavStorageUrl)) {
try {
$values[$ref] = $this->processWebDavUploadFile($form, $ref);
} catch (\Exception $e) {
// Need to catch Exception if operation on remote storage fails
$form->setErrorOn($ref, $e->getMessage());
$this->appContext->logMessage($e->getMessage(), 'lizmapadmin');
$this->appContext->logException($e, 'lizmapadmin');

return false;
}
} else {
$values[$ref] = $this->processUploadedFile($form, $ref);
}

continue;
}
Expand Down Expand Up @@ -1041,6 +1062,82 @@ protected function processUploadedFile($form, $ref)
return 'NULL';
}

/**
* Upload, delete or keep webdav file.
*
* @param \jFormsBase $form
* @param string $ref
*
* @throws \Exception
*
* @return string
*/
protected function processWebDavUploadFile($form, $ref)
{
/** @var \jFormsControlUpload2 $uploadCtrl */
$uploadCtrl = $form->getControl($ref);
$filename = $form->getData($ref);
$cnx = $this->layer->getDatasourceConnection();

$originalFile = $form->getContainer()->privateData[$ref]['originalfile'];
$newFile = $form->getContainer()->privateData[$ref]['newfile'];
$action = $form->getContainer()->privateData[$ref]['action'];

$storageUrl = $this->formControls[$ref]->webDavStorageUrl;
if (!$storageUrl) {
throw new \Exception('WEBDAV storage unavailable');
}
if ($action == 'new' && trim($filename) != '') {
// upload a new file
$newStorageUrl = $this->evaluateWebDavUrlExpression($ref, $storageUrl, $filename);
if (!$newStorageUrl) {
throw new \Exception('Invalid file path');
}
if (substr($newStorageUrl, -1) == '/') {
// it's a directory, this should't happen, maybe it's better to throw an exception
$newStorageUrl = $newStorageUrl.$filename;
}
// temp file on local file system
$newFileToCopy = $uploadCtrl->getTempFile($form->getContainer()->privateData[$ref]['newfile']);

list($uploadResult, $http_code, $uploadMessage) = RemoteStorageRequest::uploadToWebDAVStorage($newStorageUrl, $newFileToCopy);

// delete temp file
if (is_file($newFileToCopy)) {
unlink($newFileToCopy);
}

if ($uploadResult) {
return $cnx->quote($uploadResult);
}

throw new \Exception($uploadMessage);
} elseif ($originalFile !== '' && $newFile == '' && $action == 'keep') {
// keep previous file
$realStorageUrl = $this->evaluateWebDavUrlExpression($ref, $storageUrl);
if (!$realStorageUrl) {
throw new \Exception('Invalid file path');
}

return $cnx->quote($realStorageUrl.$originalFile);
} elseif ($originalFile !== '' && $newFile == '' && $action == 'del') {
// delete remote file
$realStorageUrl = $this->evaluateWebDavUrlExpression($ref, $storageUrl);
if (!$realStorageUrl) {
throw new \Exception('Invalid file path');
}
list($deleteStatus, $deleteMessage) = RemoteStorageRequest::deleteFromWebDAVStorage($realStorageUrl, $originalFile);

if ($deleteMessage) {
throw new \Exception($deleteMessage);
}

return 'NULL';
}

return 'NULL';
}

/**
* Delete the feature from the database.
*
Expand Down Expand Up @@ -1616,4 +1713,34 @@ public function evaluateExpression($expression, $form_feature = null)
$form_feature
);
}

public function evaluateWebDavUrlExpression($fieldRef, $storageUrl, $fileName = null)
{
$storageUrlFilePath = RemoteStorageRequest::getRemoteUrl($storageUrl, $fileName);
if ($storageUrlFilePath) {
// evaluate expression
$evaluatedStorageUrl = $this->evaluateExpression(array($fieldRef => $storageUrlFilePath));

if ($evaluatedStorageUrl && property_exists($evaluatedStorageUrl, $fieldRef)) {
$path = $evaluatedStorageUrl->{$fieldRef};
// ../ or ./ are not allowed
if (!preg_match('/\.\.\//', $path) && !preg_match('/\.\//', $path)) {
$profile = RemoteStorageRequest::getProfile('webdav');
if ($profile && strpos($path, $profile['baseUri']) === 0) {
if (substr($path, -1) !== '/' && !$fileName) {
// the path is a file, remove the last part of the url for delete and keep methods.
$uri_parts = explode('/', $path);
array_pop($uri_parts);

return implode('/', $uri_parts).'/';
}
// if the fileName is NOT null it means that a new upload is required, so return entire path
return $path;
}
}
}
}

return null;
}
}
17 changes: 13 additions & 4 deletions lizmap/modules/lizmap/lib/Form/QgisFormControl.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ class QgisFormControl

public $DefaultRoot;

public $isWebDAV;

public $webDavStorageUrl;

public const QGIS_NULL_VALUE = '{2839923C-8B7D-419E-B84B-CA2FE9B80EC7}';

// Table mapping QGIS and jelix forms
Expand Down Expand Up @@ -282,12 +286,12 @@ public function __construct($ref, $properties, $prop, $defaultValue, $constraint
$this->ctrl = new \jFormsControlCheckbox($this->ref);
$this->fillCheckboxValues();

break;
break;

case 'htmleditor':
$this->ctrl = new \jFormsControlHtmlEditor($this->ref);

break;
break;

case 'menulist':
case 'hidden':
Expand Down Expand Up @@ -348,6 +352,11 @@ protected function getUploadControl()
$upload->accept = $this->properties->getUploadAccept();
$upload->capture = $this->properties->getUploadCapture();
$this->DefaultRoot = $this->getEditAttribute('DefaultRoot');
// WebDAV External Resource
if ($this->getEditAttribute('StorageType') == 'WebDAV') {
$this->isWebDAV = true;
$this->webDavStorageUrl = $this->getEditAttribute('webDAVStorageUrl');
}
$this->ctrl = $upload;
}

Expand Down Expand Up @@ -751,8 +760,8 @@ public function getStoragePath($layer)
$targetFullPath = $fullPath;
}
}

if (!is_dir($targetFullPath)) {
// avoid to create local directory if the files will be stored on remote webdav server
if (!is_dir($targetFullPath) && !$this->isWebDAV) {
\jFile::createDir($targetFullPath);
}

Expand Down
4 changes: 4 additions & 0 deletions lizmap/modules/lizmap/lib/Project/Project.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1866,6 +1866,10 @@ public function getUpdatedConfig()
$layerDef = $this->getLayerDefinition($obj->id);
// Add layer type
$obj->layerType = $layerDef['type'];
// add webDav fields as layer property
if ($layerDef && array_key_exists('webDavFields', $layerDef)) {
$obj->webDavFields = $layerDef['webDavFields'];
}
// Extract layer datasource parameters only for raster/wms
if ($layerDef['type'] == 'raster' && $layerDef['provider'] == 'wms') {
// source xyz: $layerDatasource['type'] == 'xyz'
Expand Down
Loading

0 comments on commit 6cc1ba5

Please sign in to comment.