Local file storage option, democracy config, yarn package manager

curedao · May 1, 2023 · ef511ce · ef511ce
1 parent d6c2d85
commit ef511ce
Show file tree

Hide file tree

Showing 41 changed files with 17,021 additions and 37,715 deletions.
diff --git a/.gitignore b/.gitignore
@@ -43,3 +43,8 @@ Thumbs.db
 .next
 .idea/
 /.env
+
+
+# Ignore Smart Connections folder because embeddings file is large and updated frequently
+.smart-connections
+/.obsidian/
diff --git a/.mega-linter.yml b/.mega-linter.yml
@@ -0,0 +1,32 @@
+# Configuration file for MegaLinter
+# See all available variables at https://megalinter.github.io/configuration/ and in linters documentation
+
+APPLY_FIXES: all # all, none, or list of linter keys
+# ENABLE: # If you use ENABLE variable, all other languages/formats/tooling-formats will be disabled by default
+ENABLE_LINTERS:
+  [
+    #YAML_YAMLLINT,
+    #YAML_PRETTIER,
+    #SQL_SQL_LINT,
+    MARKDOWN_MARKDOWN_LINK_CHECK,
+    ENV_DOTENV_LINTER,
+    CREDENTIALS_SECRETLINT,
+    COPYPASTE_JSCPD,
+  ]
+#DISABLE:
+# - HTML
+# - COPYPASTE # Uncomment to disable checks of excessive copy-pastes
+# - SPELL # Uncomment to disable checks of spelling mistakes
+SHOW_ELAPSED_TIME: true
+FILEIO_REPORTER: true
+FILTER_REGEX_EXCLUDE: tsconfig.json
+EXCLUDED_DIRECTORIES:
+  [
+    .obsidian,
+    plugins/optomitron-real-time-decision-support-plugin/analysis-data,
+    plugins/optomitron-real-time-decision-support-plugin/analysis-data/*,
+    .git,
+    .github,
+    .vscode,
+  ]
+# DISABLE_ERRORS: true # Uncomment if you want MegaLinter to detect errors but not block CI to pass
diff --git a/.prettierignore b/.prettierignore
@@ -2,3 +2,5 @@
 
 /dist
 /coverage
+.prettierignore
+.docusaurus/
diff --git a/.secretlintrc.json b/.secretlintrc.json
@@ -0,0 +1,7 @@
+{
+  "rules": [
+    {
+      "id": "@secretlint/secretlint-rule-preset-recommend"
+    }
+  ]
+}
diff --git a/.voters.yml b/.voters.yml
@@ -0,0 +1,7 @@
+---
+# To be added to the voter list, please introduce yourself in the Introductions channel on the Discussions page.
+HermanntheGermann: 1
+kevinperrott: 1
+mikepsinn: 1
+vincenzodomina: 1
+viznov: 1
diff --git a/.voting.yml b/.voting.yml
@@ -0,0 +1,6 @@
+---
+percentageToApprove: 51 # Percentage of votes required to approve a merge
+minVotersRequired: 1 # minimum number of unique voters needed to merge.
+minVotingWindowMinutes: 720 # minimum amount of time that must pass after the pull request is updated before any
+# vote can pass. 12 hours was selected because it allows at least 4 hours of review time after subtracting 8 hours
+# for sleep.
diff --git a/.yamllint.yml b/.yamllint.yml
@@ -0,0 +1,17 @@
+---
+###########################################
+# These are the rules used for            #
+# linting all the yaml files in the stack #
+# NOTE:                                   #
+# You can disable line with:              #
+# # yamllint disable-line                 #
+###########################################
+extends: default
+rules:
+  new-lines:
+    level: warning
+    type: unix
+  line-length:
+    max: 500
+  comments:
+    min-spaces-from-content: 1 # Used to follow prettier standard: https://github.com/prettier/prettier/pull/10926
diff --git a/apps/dfda-1/app/Files/Spreadsheet/CsvFile.php b/apps/dfda-1/app/Files/Spreadsheet/CsvFile.php
@@ -14,16 +14,14 @@ class CsvFile extends AbstractSpreadsheetFile {
 	 * @param string $sourceJsonPath
 	 * @param string|null $outputPath
 	 * @return void
-	 * @throws \App\Exceptions\QMFileNotFoundException
 	 */
 	public static function jsonToCsv(string $sourceJsonPath, string $outputPath = null){
 		if(!$outputPath){
 			$outputPath = str_replace('.json', '.csv', $sourceJsonPath);
 		}
 		$sourceArray = JsonFile::getArray($sourceJsonPath);
 		$headers =  self::getHeaders($sourceArray);
-		$csv = self::arrayToCsv($headers, $sourceArray, $outputPath);
-		FileHelper::write($outputPath, $csv);
+		self::arrayToCsv($headers, $sourceArray, $outputPath);
 	}
 	public static function getDefaultFolderRelative(): string{
 		return DynamicFolder::STORAGE . "/" . static::getDefaultExtension();
@@ -59,6 +57,7 @@ private static function arrayToCsv(array $headers, array $data, string $outputPa
 			}
 			fputcsv($file, $row);
 		}
+		fclose($file);
 	}
 	private static function getHeaders(array $arr){
 		$headers = [];

diff --git a/apps/dfda-1/app/Helpers/QMHelpers.php b/apps/dfda-1/app/Helpers/QMHelpers.php
@@ -386,13 +386,11 @@ function qm_csrf_field(): HtmlString{
 function le($e, $meta = null){
     if (function_exists('xdebug_break')) {xdebug_break();}
 	//ConsoleLog::exception($e, $meta);
-		if($e instanceof \Throwable){
+	if($e instanceof \Throwable){
 		/** @var \LogicException $e */
 		throw $e;
 	}
-		if($meta){
-			$e .= QMLog::print_r($meta, true);
-		}
+	if($meta){$e .= QMLog::print_r($meta, true);}
 	throw new LogicException($e);
 }
 /**

diff --git a/apps/dfda-1/app/Slim/Controller/StaticData/GetStaticDataController.php b/apps/dfda-1/app/Slim/Controller/StaticData/GetStaticDataController.php
@@ -106,9 +106,9 @@ public static function authorized(): bool{
 		return S3Helper::loggedInUserAuthorizedForThisId($s3UserId);
 	}
 	/**
-	 * @return string
+	 * @return string|null
 	 */
-	public static function getBucketName(): string{
+	public static function getBucketName(): ?string{
 		$bucketName = QMRequest::getParam('bucket');
 		if($bucketName){
 			return $bucketName;

diff --git a/apps/dfda-1/app/Storage/DB/QMDB.php b/apps/dfda-1/app/Storage/DB/QMDB.php
@@ -518,7 +518,6 @@ public static function getPropertyModelJsonPath(): string{
 		return base_path('data/property_models.json');
 	}
 	/**
-	 * @param string $connection
 	 * @return void
 	 */
 	public static function migrateForeignKeys(): void{
@@ -531,7 +530,6 @@ public static function migrateForeignKeys(): void{
 		]);
 	}
 	/**
-	 * @param string $connection
 	 * @return void
 	 */
 	public static function migrateTables(): void{
@@ -1351,15 +1349,15 @@ public static function getTableNamesWithColumn(string $needle, bool $like = fals
 	public static function pdo(): PDO{
 		return static::db()->getPdo();
 	}
-	public static function assertLogging(){
+	public static function assertLogging(): void{
 		if(!DB::logging()){
 			le("DB NOT LOGGING!");
 		}
 	}
 	/**
 	 * @throws InvalidDatabaseCredentialsException
 	 */
-	public static function createReadonlyUser(){
+	public static function createReadonlyUser(): void{
 		$testDbCredentials = static::credentialsCommand();
 		$testDbName = static::getDbName();
 		static::exec("mysql $testDbCredentials $testDbName << EOF
@@ -1438,6 +1436,9 @@ public static function logSelectToTable(string $selectStatement, string $title =
 	public static function getExternalHost(): string{
 		return Migrations::getHost();
 	}
+	/**
+	 * @throws \App\Exceptions\NoInternetException
+	 */
 	public static function getExternalDbUrl(): string{
 		$url = static::getMySQLDbUrl();
 		$url = str_replace(static::getHost(), ThisComputer::getCurrentServerExternalIp(), $url);
@@ -3107,9 +3108,6 @@ public static function updateAllAutoIncrementSequences(): void{
             
             ');
 	}
-	/**
-	 * @throws \Doctrine\DBAL\Schema\SchemaException
-	 */
 	public static function populateDocsTable(): void {
 		$models = BaseModel::getNonAbstractModelsWithTables();
 		$json = $allTableDocs = [];

diff --git a/apps/dfda-1/app/Storage/S3/S3Helper.php b/apps/dfda-1/app/Storage/S3/S3Helper.php
@@ -37,7 +37,7 @@
 use Storage;
 abstract class S3Helper {
 	public const DISK_NAME = null;
-	protected const BUCKET = null;
+	protected const TIMEOUT = 60;
 	public const S3_REGION = 'us-east-1';
 	public const BASE_S3_PATH = null;
 	const S3CMD_CONFIG = "configs/.s3cfg";
@@ -51,10 +51,10 @@ public static function getFilenameFromUrl(string $url): string{
         $fileNameWithExtension = basename($url);
         return $fileNameWithExtension;
     }
-    /**
-     * @param string $s3Path
-     * @return int
-     */
+	/**
+	 * @param string $s3Path
+	 * @return int|null
+	 */
     public static function getSecondsSinceLastModified(string $s3Path): ?int{
         $lastModified = static::lastModified($s3Path);
         if (!$lastModified) {
@@ -101,7 +101,7 @@ public static function lastModified(string $s3Path): int {
             QMLog::infoWithoutContext(static::url($s3Path) . " last modified  " .
                 TimeHelper::timeSinceHumanString($lastModified));
         } /** @noinspection PhpRedundantCatchClauseInspection */ catch (QMFileNotFoundException | \League\Flysystem\FileNotFoundException $e) {
-            QMLog::infoWithoutContext(static::BUCKET . ": " . $e->getMessage());
+            QMLog::infoWithoutContext(static::getBucketName() . ": " . $e->getMessage());
             $lastModified = 0;
         }
         return static::$lastModified[$s3Path] = $lastModified;
@@ -137,11 +137,11 @@ public static function uploadPDF(string $s3Path, string $absoluteFilePath, bool
             throw $e;
         }
         if($result){
-            QMLog::info("Uploaded $s3Path to ".static::BUCKET);
+            QMLog::info("Uploaded $s3Path to ".static::getBucketName());
             if($deleteLocal){FileHelper::deleteFile($absoluteFilePath, __METHOD__);}
             return $s3Path;
         }
-        le("Could not upload $s3Path");throw new \LogicException();
+        le("Could not upload $s3Path");
     }
     /**
      * @param string $s3Path
@@ -165,11 +165,11 @@ public static function uploadHTML(string $s3Path, string $html, bool $log = true
         }
         if($result){
             try {
-                $url = static::getUrlForS3BucketAndPath(static::BUCKET . '/' . $s3Path);
+                $url = static::getUrlForS3BucketAndPath(static::getBucketName() . '/' . $s3Path);
             } catch (InvalidS3PathException $e) {
                 le($e);
             }
-            if($log){QMLog::info("Uploaded $s3Path to " . static::BUCKET . ". See at:\n => $url");}
+            if($log){QMLog::info("Uploaded $s3Path to " . static::getBucketName() . ". See at:\n => $url");}
             return $url;
         }
         le("Could not upload $s3Path");
@@ -188,7 +188,7 @@ public static function disk(): Filesystem{
      * @return string
      */
     public static function url(string $s3Path): string {
-        if (!static::BUCKET) {
+        if (!static::getBucketName()) {
 	        [
 		        $bucket,
 		        $s3Path,
@@ -210,7 +210,7 @@ public static function url(string $s3Path): string {
      * @return string
      */
     public static function getObjectUrl(string $s3Path): string {
-        $url = static::getS3Client()->getObjectUrl(static::BUCKET, $s3Path);
+        $url = static::getS3Client()->getObjectUrl(static::getBucketName(), $s3Path);
         return $url;
     }
     /**
@@ -305,7 +305,7 @@ public static function getLocalOrDownload(string $s3FilePath, int $maxAgeInSecon
      * @return bool
      */
     public static function existsAndNotExpired(string $s3Path, int $maxAgeInSeconds = 0): bool {
-        if (!static::BUCKET) {
+        if (!static::getBucketName()) {
 	        [
 		        $bucket,
 		        $s3Path,
@@ -340,15 +340,15 @@ public static function deleteDirectory(string $directory): bool{
         $directory = static::removeBucketFromS3PathIfNecessary($directory);
         $directory = static::prefixWithBaseS3PathIfNecessary($directory);
         $directory = str_replace('\\', '/', $directory); // Fix windows paths
-        QMLog::error("Deleting $directory from S3 bucket " . static::BUCKET);
+        QMLog::error("Deleting $directory from S3 bucket " . static::getBucketName());
         return static::disk()->deleteDirectory($directory);
     }
     /**
      * @param string $directory
      * @return bool
      */
     public static function deleteAllFilesDirectory(string $directory): bool{
-        QMLog::error("Deleting all files in $directory from S3 bucket " . static::BUCKET);
+        QMLog::error("Deleting all files in $directory from S3 bucket " . static::getBucketName());
         $disk = static::disk();
         $files = $disk->allFiles($directory);
         return $disk->delete($files);
@@ -362,7 +362,7 @@ public static function softDeleteDirectory(string $directory){
         le("Not implemented yet!");
         /** @noinspection PhpUnreachableStatementInspection */
         $directory = static::prefixWithBaseS3PathIfNecessary($directory);
-        QMLog::error("Deleting $directory from S3 bucket " . static::BUCKET);
+        QMLog::error("Deleting $directory from S3 bucket " . static::getBucketName());
         $disk = static::disk();
         $deletedDir = 'deleted/'.$directory;
         $disk->makeDirectory($deletedDir);
@@ -534,15 +534,15 @@ public static function delete(string $s3Path): bool {
 		    le("Could not delete $s3Path!  Make sure you don't have versioning turned on.  Error: " . $e->getMessage());
 		}
     }
-    /**
-     * @param string $s3Path
-     * @return mixed|string
-     */
+	/**
+	 * @param string $s3Path
+	 * @return string
+	 */
     protected static function removeBucketFromS3PathIfNecessary(string $s3Path): string{
         $s3Path = QMStr::after(S3Private::getBucketName() . '/', $s3Path, $s3Path);
         $s3Path = QMStr::after(S3Public::getBucketName() . '/', $s3Path, $s3Path);
-        if(static::BUCKET){
-            $s3Path = QMStr::after(static::BUCKET . '/', $s3Path, $s3Path);
+        if(static::getBucketName()){
+            $s3Path = QMStr::after(static::getBucketName() . '/', $s3Path, $s3Path);
         }
         return $s3Path;
     }
@@ -666,7 +666,7 @@ public static function uploadFile(string $s3Path, string $path, bool $overwrite)
             return null;
         }
         $path = FileHelper::absPath($path);
-        \App\Logging\ConsoleLog::info("Uploading $path to $s3Path in ".static::BUCKET);
+        \App\Logging\ConsoleLog::info("Uploading $path to $s3Path in ".static::getBucketName());
         try {
             static::put($s3Path, file_get_contents($path));
             return static::url($s3Path);
@@ -780,11 +780,11 @@ public static function uploadObject(string $key, $value): string{
         }
         if($result){
             try {
-                $url = static::getUrlForS3BucketAndPath(static::BUCKET . '/' . $s3Path);
+                $url = static::getUrlForS3BucketAndPath(static::getBucketName() . '/' . $s3Path);
             } catch (InvalidS3PathException $e) {
                 le($e);
             }
-            QMLog::info("Uploaded $s3Path to " . static::BUCKET . ". See at:\n => $url");
+            QMLog::info("Uploaded $s3Path to " . static::getBucketName() . ". See at:\n => $url");
             return $url;
         }
         le("Could not upload $s3Path");
@@ -816,6 +816,7 @@ public static function bucketToDisk(string $bucket): string {
         $map = [
             S3Public::getBucketName() => S3Public::DISK_NAME,
             S3Private::getBucketName() => S3Private::DISK_NAME,
+	        S3PrivateGlobal::getBucketName() => S3PrivateGlobal::DISK_NAME,
         ];
         return $map[$bucket];
     }
@@ -952,5 +953,22 @@ public static function downloadIfNotExists(string $s3Path, string $relativeDownl
 		if(FileHelper::fileExists($absoluteDownloadPath)){return;}
 		static::download($s3Path, $absoluteDownloadPath);
 	}
-	abstract public static function getBucketName(): string;
+	abstract public static function getBucketName(): ?string;
+	abstract protected static function getLocalFileSystemConfig(): array;
+	public static function getConfig(): array{
+		$bucket = static::getBucketName();
+		if(!$bucket){
+			return static::getLocalFileSystemConfig();
+		}
+		return [
+			'driver' => 's3',
+			'key' => env('STORAGE_ACCESS_KEY_ID'),
+			'secret' => env('STORAGE_SECRET_ACCESS_KEY'),
+			'region' => static::S3_REGION,
+			'bucket' => env('STORAGE_BUCKET_PRIVATE', $bucket),
+			'url' => env('STORAGE_URL'),
+			'endpoint' => env('STORAGE_ENDPOINT'),
+			'options' => ['timeout' => static::TIMEOUT,],
+		];
+	}
 }
diff --git a/apps/dfda-1/app/Storage/S3/S3Images.php b/apps/dfda-1/app/Storage/S3/S3Images.php
@@ -23,7 +23,7 @@ public static function uploadImage(string $key, $imageData, string $type) {
         }
         $s3Path = $key . '.' . $type;
         $options = [
-            'Bucket'      => static::BUCKET,
+            'Bucket'      => static::getBucketName(),
             'Key'         => $s3Path,
             'Body'        => $imageData,
             'ContentType' => $contentType,