From bc39916fd310a16a52e7b8e11ac2c4d765a0ff3a Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Mon, 13 Jan 2020 16:40:27 +0530 Subject: [PATCH 1/9] updated autodiscovery and auto download commands for self-updates added new columns --- README.md | 9 + composer.json | 12 +- src/Geo.php | 171 +++++++------ src/GeoController.php | 193 +++++++++----- src/GeoServiceProvider.php | 33 +-- src/commands/BuildPplTree.php | 153 ++++++++++++ src/commands/Download.php | 59 +++++ src/commands/helpers/geoCollection.php | 54 ++-- src/commands/helpers/geoItem.php | 100 ++++---- src/commands/seedGeoFile.php | 276 +++++++++++++------- src/commands/seedJsonFile.php | 58 +++-- src/commands/truncTable.php | 19 +- src/dbTree/EloquentTreeItem.php | 131 +++++----- src/migrations/2017_02_13_090952_geo.php | 20 +- src/routes.php | 16 +- src/vue/geo-select.vue | 305 +++++++++++++---------- 16 files changed, 1028 insertions(+), 581 deletions(-) create mode 100644 src/commands/BuildPplTree.php create mode 100644 src/commands/Download.php diff --git a/README.md b/README.md index 57e2bbb..1d7010c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ What you dont get: `composer require igaster/laravel_cities` +this package uses autodiscover or else you can add manually + - Add Service Provider in app.php: ```php @@ -41,6 +43,13 @@ wget http://download.geonames.org/export/dump/allCountries.zip && unzip allCount wget http://download.geonames.org/export/dump/hierarchy.zip && unzip hierarchy.zip && rm hierarchy.zip ``` +or otherwise you can use +``` +artisan geo:download +``` + +Download a *.txt files from geonames.org By default it will download allcountries and hierarchy files otherwise you can pass flag --countries for specific countries + - Migrate and Seed. Run: ``` diff --git a/composer.json b/composer.json index 32b6439..b926c6e 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,17 @@ } ], "require": { - "illuminate/contracts": "~5.1 || ^6.0" + "illuminate/contracts": "~5.5 || ^6.0" + }, + "extra": { + "branch-alias": { + "dev-master": "3.2-dev" + }, + "laravel": { + "providers": [ + "Igaster\\LaravelCities\\GeoServiceProvider" + ] + } }, "require-dev": { "orchestra/testbench": "^4.0", diff --git a/src/Geo.php b/src/Geo.php index 3903565..e45e997 100644 --- a/src/Geo.php +++ b/src/Geo.php @@ -1,13 +1,14 @@ - 'array', - ]; + protected $casts = ['alternames' => 'array']; // Hide From JSON - protected $hidden = [ - 'alternames', - 'left', - 'right', - 'depth', - ]; - + protected $hidden = ['alternames', 'left', 'right', 'depth']; // ---------------------------------------------- // Scopes // ---------------------------------------------- - public function scopeCountry($query, $countryCode){ + public function scopeCountry($query, $countryCode) + { return $query->where('country', $countryCode); } - public function scopeCapital($query){ - return $query->where('level',Geo::LEVEL_CAPITAL); + public function scopeCapital($query) + { + return $query->where('level', self::LEVEL_CAPITAL); } - public function scopeLevel($query,$level){ - return $query->where('level',$level); + public function scopeLevel($query, $level) + { + return $query->where('level', $level); } - public function scopeDescendants($query){ + public function scopeDescendants($query) + { return $query->where('left', '>', $this->left)->where('right', '<', $this->right); } - public function scopeAncenstors($query){ - return $query->where('left','<', $this->left)->where('right', '>', $this->right); + public function scopeAncenstors($query) + { + return $query->where('left', '<', $this->left)->where('right', '>', $this->right); } - public function scopeChildren($query){ - return $query->where(function($query) - { + public function scopeChildren($query) + { + return $query->where(function ($query) { $query->where('left', '>', $this->left) ->where('right', '<', $this->right) - ->where('depth', $this->depth+1); - }); + ->where('depth', $this->depth + 1); + }); } - public function scopeSearch($query,$search){ - $search = '%'.mb_strtolower($search).'%'; + public function scopeSearch($query, $search) + { + $search = '%' . mb_strtolower($search) . '%'; - return $query->where(function($query) use($search) - { + return $query->where(function ($query) use ($search) { $query->whereRaw('LOWER(alternames) LIKE ?', [$search]) ->orWhereRaw('LOWER(name) LIKE ?', [$search]); }); - } - public function scopeAreDescentants($query,Geo $parent){ - return $query->where(function($query) use($parent) - { + public function scopeAreDescentants($query, Geo $parent) + { + return $query->where(function ($query) use ($parent) { $query->where('left', '>', $parent->left) ->where('right', '<', $parent->right); }); } - public function scopeTest($query){ + public function scopeTest($query) + { return $query; } @@ -90,23 +87,23 @@ public function scopeTest($query){ // ---------------------------------------------- // public function setXxxAttribute($value){ - // $this->attributes['xxx'] = $value; + // $this->attributes['xxx'] = $value; // } // ---------------------------------------------- // Relations // ---------------------------------------------- - // ---------------------------------------------- // Methods // ---------------------------------------------- // search in `name` and `alternames` / return collection - public static function searchNames($name, Geo $parent =null){ + public static function searchNames($name, Geo $parent = null) + { $query = self::search($name)->orderBy('name', 'ASC'); - if ($parent){ + if ($parent) { $query->areDescentants($parent); } @@ -114,81 +111,91 @@ public static function searchNames($name, Geo $parent =null){ } // get all Countries - public static function getCountries(){ - return self::level(Geo::LEVEL_COUNTRY)->orderBy('name')->get(); + public static function getCountries() + { + return self::level(self::LEVEL_COUNTRY)->orderBy('name')->get(); } // get Country by country Code (eg US,GR) - public static function getCountry($countryCode){ - return self::level(Geo::LEVEL_COUNTRY)->country($countryCode)->first(); + public static function getCountry($countryCode) + { + return self::level(self::LEVEL_COUNTRY)->country($countryCode)->first(); } // get multiple item by Ids - public static function getByIds(array $Ids = []){ - return self::whereIn('id',$Ids)->orderBy('name')->get(); + public static function getByIds(array $Ids = []) + { + return self::whereIn('id', $Ids)->orderBy('name')->get(); } // is imediate Child of $item ? - public function isChildOf(Geo $item){ - return ($this->left > $item->left) && ($this->right < $item->right) && ($this->depth == $item->depth+1); + public function isChildOf(Geo $item) + { + return ($this->left > $item->left) && ($this->right < $item->right) && ($this->depth == $item->depth + 1); } // is imediate Parent of $item ? - public function isParentOf(Geo $item){ - return ($this->left < $item->left) && ($this->right > $item->right) && ($this->depth == $item->depth-1); + public function isParentOf(Geo $item) + { + return ($this->left < $item->left) && ($this->right > $item->right) && ($this->depth == $item->depth - 1); } // is Child of $item (any depth) ? - public function isDescendantOf(Geo $item){ + public function isDescendantOf(Geo $item) + { return ($this->left > $item->left) && ($this->right < $item->right); } // is Parent of $item (any depth) ? - public function isAncenstorOf(Geo $item){ + public function isAncenstorOf(Geo $item) + { return ($this->left < $item->left) && ($this->right > $item->right); } - // retrieve by name - public static function findName($name){ - return self::where('name',$name)->first(); + // retrieve by name + public static function findName($name) + { + return self::where('name', $name)->first(); } // get all imediate Children (Collection) - public function getChildren(){ - return self::descendants()->where('depth', $this->depth+1)->orderBy('name')->get(); + public function getChildren() + { + return self::descendants()->where('depth', $this->depth + 1)->orderBy('name')->get(); } // get Parent (Geo) - public function getParent(){ - return self::ancenstors()->where('depth', $this->depth-1)->first(); + public function getParent() + { + return self::ancenstors()->where('depth', $this->depth - 1)->first(); } // get all Ancnstors (Collection) ordered by level (Country -> City) - public function getAncensors(){ + public function getAncensors() + { return self::ancenstors()->orderBy('depth')->get(); } // get all Descendants (Collection) Alphabetical - public function getDescendants(){ + public function getDescendants() + { return self::descendants()->orderBy('level')->orderBy('name')->get(); } - - - // Return only $fields as Json. null = Show all - public function fliterFields($fields = null){ - - if (is_string($fields)){ // Comma Seperated List (eg Url Param) + // Return only $fields as Json. null = Show all + public function fliterFields($fields = null) + { + if (is_string($fields)) { // Comma Seperated List (eg Url Param) $fields = explode(',', $fields); } - if(empty($fields)){ + if (empty($fields)) { $this->hidden = []; } else { - $this->hidden = ['id','parent_id','left','right','depth','name','alternames','country','level','population','lat','long']; + $this->hidden = ['id', 'parent_id', 'left', 'right', 'depth', 'name', 'alternames', 'country', 'level', 'population', 'lat', 'long']; foreach ($fields as $field) { $index = array_search($field, $this->hidden); - if($index !== false){ + if ($index !== false) { unset($this->hidden[$index]); } }; @@ -202,16 +209,8 @@ public function fliterFields($fields = null){ // Routes // ---------------------------------------------- - public static function ApiRoutes(){ - Route::group(['prefix' => 'geo'], function(){ - Route::get('search/{name}/{parent_id?}', '\Igaster\LaravelCities\GeoController@search'); - Route::get('item/{id}', '\Igaster\LaravelCities\GeoController@item'); - Route::get('items/{ids}', '\Igaster\LaravelCities\GeoController@items'); - Route::get('children/{id}', '\Igaster\LaravelCities\GeoController@children'); - Route::get('parent/{id}', '\Igaster\LaravelCities\GeoController@parent'); - Route::get('country/{code}', '\Igaster\LaravelCities\GeoController@country'); - Route::get('countries', '\Igaster\LaravelCities\GeoController@countries'); - }); + public static function ApiRoutes() + { + require_once __DIR__ . '/routes.php'; } - -} \ No newline at end of file +} diff --git a/src/GeoController.php b/src/GeoController.php index cd2093d..59081ac 100644 --- a/src/GeoController.php +++ b/src/GeoController.php @@ -1,82 +1,141 @@ -has('fields')){ +class GeoController extends Controller +{ + // [Geo] Get an item by $id + public function item($id) + { + $geo = Geo::find($id); + $this->applyFilter($geo); + return Response::json($geo); + } - if(get_class($geo) == \Illuminate\Database\Eloquent\Collection::class){ - foreach ($geo as $item) { - $this->applyFilter($item); - }; - return $geo; - } + // [Collection] Get multiple items by ids (comma seperated string or array) + public function items($ids = []) + { + if (is_string($ids)) { + $ids = explode(',', $ids); + } + + $items = Geo::getByIds($ids); + return Response::json($items); + } + + // [Collection] Get children of $id + public function children($id) + { + return $this->applyFilter(Geo::find($id)->getChildren()); + } + + // [Geo] Get parent of $id + public function parent($id) + { + $geo = Geo::find($id)->getParent(); + $this->applyFilter($geo); + return Response::json($geo); + } + + // [Geo] Get country by $code (two letter code) + public function country($code) + { + $geo = Geo::getCountry($code); + $this->applyFilter($geo); + return Response::json($geo); + } + + // [Collection] Get all countries + public function countries() + { + return $this->applyFilter(Geo::level(Geo::LEVEL_COUNTRY)->get()); + } + + // [Collection] Search for %$name% in 'name' and 'alternames'. Optional filter to children of $parent_id + public function search($name, $parent_id = null) + { + if ($parent_id) { + return $this->applyFilter(Geo::searchNames($name, Geo::find($parent_id))); + } + return $this->applyFilter(Geo::searchNames($name)); + } + + public function ancestors($id) + { + $current = Geo::find($id); + $ancestors = $current->ancenstors()->get()->sortBy('a1code')->values(); + $ancestors->push($current); + + $result = collect(); + foreach ($ancestors as $i => $ancestor) { + if ($i === 0) { + $locations = Geo::getCountries(); + } else { + $parent = $ancestor->getParent(); + if (! $parent) { + continue; + } + + $locations = $parent->getChildren(); + } + + $selected = $locations->firstWhere('id', $ancestor->id); + $selected && $selected->isSelected = true; + $result->push($locations); + + if ($i == $ancestors->count() - 1 && $ancestor) { + $childrens = $ancestor->getChildren(); + if ($childrens->count()) { + $result->push($childrens); + } + } + } + + $result = $this->applyFilter($result); + + return $result; + } + + public function breadcrumbs($id) + { + $current = Geo::find($id); + $ancestors = $current->ancenstors()->get(); + $ancestors->push($current); + + $ancestors = $this->applyFilter($ancestors); + + return $ancestors; + } + + // Apply Filter from request to json representation of an item or a collection + // api/call?fields=field1,field2 + protected function applyFilter($geo) + { + if (request()->has('fields')) { + if (get_class($geo) == Collection::class) { + foreach ($geo as $item) { + $this->applyFilter($item); + } + + return $geo; + } $fields = request()->input('fields'); - if($fields == 'all'){ + if ($fields == 'all') { $geo->fliterFields(); } else { $fields = explode(',', $fields); - array_walk($fields, function(&$item){ + array_walk($fields, function (&$item) { $item = strtolower(trim($item)); }); $geo->fliterFields($fields); } - } - return $geo; - } - - // [Geo] Get an item by $id - public function item($id){ - $geo = Geo::find($id); - $this->applyFilter($geo); - return \Response::json($geo); - } - - // [Collection] Get multiple items by ids (comma seperated string or array) - public function items($ids = []){ - if(is_string($ids)){ - $ids=explode(',', $ids); - } - - $items = Geo::getByIds($ids); - return \Response::json($items); - } - - // [Collection] Get children of $id - public function children($id){ - return $this->applyFilter(Geo::find($id)->getChildren()); - } - - // [Geo] Get parent of $id - public function parent($id){ - $geo = Geo::find($id)->getParent(); - $this->applyFilter($geo); - return \Response::json($geo); - } - - // [Geo] Get country by $code (two letter code) - public function country($code){ - $geo = Geo::getCountry($code); - $this->applyFilter($geo); - return \Response::json($geo); - } - - // [Collection] Get all countries - public function countries(){ - return $this->applyFilter(Geo::level(Geo::LEVEL_COUNTRY)->get()); - } - - // [Collection] Search for %$name% in 'name' and 'alternames'. Optional filter to children of $parent_id - public function search($name,$parent_id = null){ - if ($parent_id) - return $this->applyFilter(Geo::searchNames($name, Geo::find($parent_id))); - else - return $this->applyFilter(Geo::searchNames($name)); - } + } + return $geo; + } } diff --git a/src/GeoServiceProvider.php b/src/GeoServiceProvider.php index de1fc48..9344e4b 100644 --- a/src/GeoServiceProvider.php +++ b/src/GeoServiceProvider.php @@ -1,44 +1,31 @@ -loadMigrationsFrom(__DIR__.'/migrations'); + $this->loadMigrationsFrom(__DIR__ . '/migrations'); // Load Routes // $this->loadRoutesFrom(__DIR__.'/routes.php'); $this->publishes([ - __DIR__.'/vue' => resource_path('LaravelCities'), + __DIR__ . '/vue' => resource_path('LaravelCities'), ], 'vue'); - // Register Commands if ($this->app->runningInConsole()) { $this->commands([ \Igaster\LaravelCities\commands\seedGeoFile::class, \Igaster\LaravelCities\commands\seedJsonFile::class, + \Igaster\LaravelCities\commands\BuildPplTree::class, + \Igaster\LaravelCities\commands\Download::class, ]); } - } - -} \ No newline at end of file +} diff --git a/src/commands/BuildPplTree.php b/src/commands/BuildPplTree.php new file mode 100644 index 0000000..8c02692 --- /dev/null +++ b/src/commands/BuildPplTree.php @@ -0,0 +1,153 @@ +option('countries'); + $countries = explode(',', $countries); + + try { + $this->downloadAdmin1CodesASCIIIfNotExists(); + foreach ($countries as $country) { + //$this->determinePplParentId($country); + $this->buildPplHierarchy($country); + $this->mergeHierarchies($country); + } + + //$this->mergeAllHierarchies($countries); + } catch (\Exception $e) { + $this->error($e->getMessage()); + } + } + + private function downloadAdmin1CodesASCIIIfNotExists() + { + $fileName = 'admin1CodesASCII.txt'; + $localPath = storage_path("geo/$fileName"); + $remotePath = "http://download.geonames.org/export/dump/$fileName"; + if (! file_exists($localPath)) { + if (! copy($remotePath, $localPath)) { + throw new \Exception("Failed to download the file $remotePath"); + } + } + } + + private function determinePplParentId($country) + { + $geos = \DB::table('geo') + ->select(['id', 'name', 'country', 'a1code', 'level']) + ->where('country', $country) + ->whereRaw('parent_id IS NULL') + ->get(); + + $map = $this->mapAdmin1Codes(); + + foreach ($geos as $geo) { + $key = "{$geo->country}.{$geo->a1code}"; + $parentId = Arr::get($map, $key); + + if (! $parentId) { + $this->warn("Not found parent for: {$geo->id} ({$geo->name})"); + continue; + } + + \DB::table('geo')->where('id', $geo->id)->update(['parent_id' => $parentId]); + } + } + + private function buildPplHierarchy($country) + { + $hierarchyPplFilePath = storage_path("geo/hierarchy-ppl-$country.txt"); + $countryFilePath = storage_path("geo/$country.txt"); + + $map = $this->mapAdmin1Codes(); + + $rows = ''; + foreach (file($countryFilePath) as $line) { + $cols = explode("\t", trim($line)); + + if (strpos($cols[7], 'PPL') === false) { + continue; + } + + $geoId = $cols[0]; + $key = "{$cols[8]}.{$cols[10]}"; + $geoParentId = Arr::get($map, $key); + + if ($geoParentId) { + $rows .= "$geoParentId\t$geoId" . PHP_EOL; + } + } + + file_put_contents($hierarchyPplFilePath, $rows); + } + + private function mergeHierarchies($country) + { + $files = [ + storage_path('geo/hierarchy-origin.txt'), + storage_path("geo/hierarchy-ppl-$country.txt"), + ]; + + $lines = []; + foreach ($files as $file) { + foreach (file($file) as $line) { + $lines[] = trim($line); + } + } + $lines = array_unique($lines); + + $content = implode(PHP_EOL, $lines); + + file_put_contents(storage_path("geo/hierarchy-$country.txt"), $content); + } + + private function mergeAllHierarchies($countries) + { + $files = [ + storage_path('geo/hierarchy-origin.txt') + ]; + + foreach ($countries as $country) { + $files[] = storage_path("geo/hierarchy-ppl-$country.txt"); + } + + $lines = []; + foreach ($files as $file) { + foreach (file($file) as $line) { + $lines[] = trim($line); + } + } + $lines = array_unique($lines); + + $content = implode(PHP_EOL, $lines); + + file_put_contents(storage_path('geo/hierarchy.txt'), $content); + } + + private function mapAdmin1Codes() + { + $map = []; // @example: UA.01 => $parent_id + + $fileName = 'admin1CodesASCII.txt'; + $localPath = storage_path("geo/$fileName"); + + foreach (file($localPath) as $line) { + $cols = explode("\t", trim($line)); + $key = $cols[0]; + $parentId = $cols[3]; + $map[$key] = $parentId; + } + + return $map; + } +} diff --git a/src/commands/Download.php b/src/commands/Download.php new file mode 100644 index 0000000..8904dcc --- /dev/null +++ b/src/commands/Download.php @@ -0,0 +1,59 @@ +option('countries'); + + $files = ['hierarchy.zip', 'admin1CodesASCII.txt']; + + if ($countries == self::ALL_COUNTRIES) { + $files = ['allCountries.zip', 'hierarchy.zip']; + } else { + $countries = explode(',', $countries); + + foreach ($countries as $country) { + $files[] = "$country.zip"; + } + } + + return $files; + } + + public function handle() + { + foreach ($this->getFileNames() as $fileName) { + $source = "http://download.geonames.org/export/dump/$fileName"; + $target = storage_path("geo/$fileName"); + $targetTxt = storage_path('geo/' . preg_replace('/\.zip/', '.txt', $fileName)); + + $this->info(" Source file {$source}" . PHP_EOL . " Target file {$targetTxt}"); + + if (! (file_exists($target) || file_exists($targetTxt))) { + $this->info(" Downloading file {$fileName}"); + if (! copy($source, $target)) { + throw new \Exception("Failed to download the file $source"); + } + } + + if (file_exists($target) && ! file_exists($targetTxt)) { + if (preg_match('/\.zip/', $fileName)) { + $zip = new \ZipArchive; + $zip->open($target); + $zip->extractTo(dirname($target)); + $zip->close(); + } + } + } + } +} diff --git a/src/commands/helpers/geoCollection.php b/src/commands/helpers/geoCollection.php index 55cd38d..f7d63c0 100644 --- a/src/commands/helpers/geoCollection.php +++ b/src/commands/helpers/geoCollection.php @@ -1,30 +1,38 @@ -items[$item->getId()] = $item; - } +class geoCollection +{ + public $items = []; - public function findGeoId($geoId){ - return isset($this->items[$geoId]) ? $this->items[$geoId] : null; - } + public function add($item) + { + $this->items[$item->getId()] = $item; + } - public function findId($id){ - foreach ($this->items as $item) { - if($item->getId() == $id) - return $item; - } - return false; - } + public function findGeoId($geoId) + { + return isset($this->items[$geoId]) ? $this->items[$geoId] : null; + } + public function findId($id) + { + foreach ($this->items as $item) { + if ($item->getId() == $id) { + return $item; + } + } + return false; + } - public function findName($name){ - foreach ($this->items as $item) { - if($item->data[2] == $name) - return $item; - } - return false; - } + public function findName($name) + { + foreach ($this->items as $item) { + if ($item->data[2] == $name) { + return $item; + } + } + return false; + } } diff --git a/src/commands/helpers/geoItem.php b/src/commands/helpers/geoItem.php index 499e875..f613669 100644 --- a/src/commands/helpers/geoItem.php +++ b/src/commands/helpers/geoItem.php @@ -1,47 +1,55 @@ -data=$rawData; - $this->geoItems=$geoItems; - } - - public function getId(){ - return $this->data[0]; - } - - public function getName(){ - return $this->data[2]; - } - - public function setParent($geoId){ - if( $parent = $this->geoItems->findGeoId($geoId)){ - $this->parentId = $geoId; - } - } - - public function addChild($geoId){ - $this->childrenGeoId[] = $geoId; - } - - public function getChildren(){ - $results = []; - foreach ($this->childrenGeoId as $geoId) { - $results[] = $this->geoItems->findGeoId($geoId); - } - return $results; - } +data = $rawData; + $this->geoItems = $geoItems; + } + + public function getId() + { + return $this->data[0]; + } + + public function getName() + { + return $this->data[2]; + } + + public function setParent($geoId) + { + if ($parent = $this->geoItems->findGeoId($geoId)) { + $this->parentId = $geoId; + } + } + + public function addChild($geoId) + { + $this->childrenGeoId[] = $geoId; + } + + public function getChildren() + { + $results = []; + foreach ($this->childrenGeoId as $geoId) { + $results[] = $this->geoItems->findGeoId($geoId); + } + return $results; + } } diff --git a/src/commands/seedGeoFile.php b/src/commands/seedGeoFile.php index 56fb6f7..f66393e 100644 --- a/src/commands/seedGeoFile.php +++ b/src/commands/seedGeoFile.php @@ -1,9 +1,15 @@ -driver = strtolower(config("database.connections.{$connection}.driver")); - $this->pdo = \DB::connection()->getPdo(\PDO::FETCH_ASSOC); - if (!\Schema::hasTable('geo')) - return; + $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); - $this->geoItems = new geoCollection(); + if (! Schema::hasTable('geo')) { + return; + } } - public function sql($sql){ + public function sql($sql) + { $result = $this->pdo->query($sql); - if($result === false) - throw new Exception("Error in SQL : '$sql'\n".PDO::errorInfo(), 1); - + if ($result === false) { + throw new Exception("Error in SQL : '$sql'\n" . PDO::errorInfo(), 1); + } + return $result->fetch(); - } + } - public function buildDbTree($item, $count = 1, $depth = 0){ - $item->left=$count++; - $item->depth=$depth; + public function buildDbTree($item, $count = 1, $depth = 0) + { + $item->left = $count++; + $item->depth = $depth; foreach ($item->getChildren() as $child) { - $count = $this->buildDbTree($child, $count, $depth+1); + $count = $this->buildDbTree($child, $count, $depth + 1); } - $item->right=$count++; + $item->right = $count++; + return $count; } - - public function printTree($item){ - $levelStr= str_repeat('--', $item->depth); - $this->info(sprintf("%s %s [%d,%d]", $levelStr, $item->getName(),$item->left,$item->right)); - foreach ($item->getChildren() as $child) + + public function printTree($item) + { + $levelStr = str_repeat('--', $item->depth); + $this->info(sprintf('%s %s [%d,%d]', $levelStr, $item->getName(), $item->left, $item->right)); + foreach ($item->getChildren() as $child) { $this->printTree($child); + } } - public function handle() { - $start = microtime(true); + /** + * Get fully qualified table name with prefix if any + * + * @return string + */ + public function getFullyQualifiedTableName() : string + { + return DB::getTablePrefix() . 'geo'; + } - $fileName = $this->argument('country') ? strtoupper($this->argument('country')) : 'allCountries'; - $fileName = storage_path("geo/{$fileName}.txt"); - $append = $this->option('append'); + protected function getColumnsAsStringDelimated($delimeter = '"', bool $onlyPrefix = false) + { + $columns = [ + 'id', 'parent_id', 'left', 'right', 'depth', 'name', 'alternames', 'country', 'a1code', 'level', 'population', 'lat', 'long', 'timezone', + ]; - // Read Raw file + $modifiedColumns = []; + + foreach($columns as $column) { + $modifiedColumns[] = $delimeter . $column . (($onlyPrefix) ? '' : $delimeter); + } + + return implode(',', $modifiedColumns); + } + + public function getDBStatement() : array + { + + $sql = "INSERT INTO {$this->getFullyQualifiedTableName()} ( {$this->getColumnsAsStringDelimated()} ) VALUES ( {$this->getColumnsAsStringDelimated(':', true)} )"; + + if ($this->driver == 'mysql') { + $sql = "INSERT INTO {$this->getFullyQualifiedTableName()} ( {$this->getColumnsAsStringDelimated('`')} ) VALUES ( {$this->getColumnsAsStringDelimated(':', true)} )"; + } + + return [$this->pdo->prepare($sql), $sql]; + } + + public function readFile(string $fileName) + { $this->info("Reading File '$fileName'"); $filesize = filesize($fileName); $handle = fopen($fileName, 'r'); $count = 0; - $progressBar = new \Symfony\Component\Console\Helper\ProgressBar($this->output, 100); + $progressBar = new ProgressBar($this->output, 100); + while (($line = fgets($handle)) !== false) { // ignore empty lines and comments - if ( ! $line or $line === '' or strpos($line, '#') === 0) continue; + if (! $line || $line === '' || strpos($line, '#') === 0) { + continue; + } // Convert TAB sepereted line to array $line = explode("\t", $line); // Check for errors - if(count($line)!== 19) dd($line[0],$line[2]); + if (count($line) !== 19) { + dd($line[0], $line[2]); + } switch ($line[7]) { case 'PCLI': // Country case 'PPLC': // Capital case 'ADM1': case 'ADM2': - case 'ADM3': + case 'ADM3': // 8 sec + case 'PPLA': // областные центры + case 'PPLA2': // Корсунь + //case 'PPL': // Яблунівка + // 185 sec $this->geoItems->add(new geoItem($line, $this->geoItems)); $count++; break; } - $progress = ftell($handle)/$filesize*100; + $progress = ftell($handle) / $filesize * 100; $progressBar->setProgress($progress); } + $progressBar->finish(); + $this->info(" Finished Reading File. $count items loaded"); + } + + public function handle() + { + $this->geoItems = new geoCollection(); + + $start = microtime(true); + $country = strtoupper($this->argument('country')); + $sourceName = $country ? $country : 'allCountries'; + $fileName = storage_path("geo/{$sourceName}.txt"); + $isAppend = $this->option('append'); + + $this->info("Start seeding for $country"); + + // Read Raw file + $this->readFile($fileName); // Read hierarchy + $this->readHierarcy($country); + + // Build Tree + $this->buildTree(); + + // Clear Table + if (! $isAppend) { + $this->info("Truncating '{$this->getFullyQualifiedTableName()}' table..."); + DB::table('geo')->truncate(); + } + + // Store Tree in DB + $this->writeToDb(); + + //Lets get back FOREIGN_KEY_CHECKS to laravel + DB::statement('SET FOREIGN_KEY_CHECKS=1;'); + $this->info(PHP_EOL . ' Relation checks enabled'); + + $this->info(' Done'); + $time_elapsed_secs = microtime(true) - $start; + $this->info("Timing: $time_elapsed_secs sec"); + } + + public function readHierarcy(string $country) + { + //if all countries $fileName = storage_path('geo/hierarchy.txt'); + //ini_set('xdebug.max_nesting_level', 5000); + if ($country != '') { + $fileName = storage_path("geo/hierarchy-$country.txt"); + } + $this->info("Opening File '$fileName'"); $handle = fopen($fileName, 'r'); $filesize = filesize($fileName); $count = 0; - $progressBar = new \Symfony\Component\Console\Helper\ProgressBar($this->output, 100); + $progressBar = new ProgressBar($this->output, 100); while (($line = fgetcsv($handle, 0, "\t")) !== false) { - $parent = $item=$this->geoItems->findGeoId($line[0]); - $child = $item=$this->geoItems->findGeoId($line[1]); + $parent = $this->geoItems->findGeoId($line[0]); + $child = $this->geoItems->findGeoId($line[1]); - if( $parent !== null && $child !== null){ + if ($parent !== null && $child !== null) { $parent->addChild($line[1]); $child->setParent($line[0]); $count++; } - $progress = ftell($handle)/$filesize*100; + + $progress = ftell($handle) / $filesize * 100; $progressBar->setProgress($progress); } $this->info(" Hierarcy building completed. $count items loaded"); + } - // Build Tree - $count = 0; $countOrphan = 0; - $sql = 'SELECT MAX("right") as maxRight FROM geo'; + public function buildTree() + { + $count = 0; + $countOrphan = 0; + $sql = 'SELECT MAX("right") as maxRight FROM ' . $this->getFullyQualifiedTableName(); $result = $this->sql($sql); - $maxBoundary = (isset($result['maxRight']) && is_numeric($result['maxRight'])) ? $result['maxRight']+1 : 0; + $maxBoundary = (isset($result['maxRight']) && is_numeric($result['maxRight'])) ? $result['maxRight'] + 1 : 0; + foreach ($this->geoItems->items as $item) { - if($item->parentId === null){ - - if($item->data[7] !== 'PCLI'){ + if ($item->parentId === null) { + if ($item->data[7] !== 'PCLI') { // $this->info("- Skiping Orphan {$item->data[2]} #{$item->data[0]}"); $countOrphan++; continue; @@ -129,63 +235,53 @@ public function handle() { $count++; $this->info("+ Building Tree for Country: {$item->data[2]} #{$item->data[0]}"); - $maxBoundary=$this->buildDbTree($item,$maxBoundary,0); + $maxBoundary = $this->buildDbTree($item, $maxBoundary, 0); // $this->printTree($item,$output); } } - $this->info("Finished: {$count} Countries imported. $countOrphan orphan items skiped"); + $this->info("Finished: {$count} Countries imported. $countOrphan orphan items skiped"); + } - // Empty Table - if (!$append){ - $this->info("Truncating 'geo' table..."); - \DB::table('geo')->truncate(); - } - + public function writeToDb() + { // Store Tree in DB - $this->info("Writing in Database"); + $this->info('Writing in Database'); - if ($this->driver == 'mysql') { - $stmt = $this->pdo->prepare("INSERT INTO geo (`id`, `parent_id`, `left`, `right`, `depth`, `name`, `alternames`, `country`, `level`, `population`, `lat`, `long`) VALUES (:id, :parent_id, :left, :right, :depth, :name, :alternames, :country, :level, :population, :lat, :long)"); - } else { - $stmt = $this->pdo->prepare("INSERT INTO geo (\"id\", \"parent_id\", \"left\", \"right\", \"depth\", \"name\", \"alternames\", \"country\", \"level\", \"population\", \"lat\", \"long\") VALUES (:id, :parent_id, :left, :right, :depth, :name, :alternames, :country, :level, :population, :lat, :long)"); - } + [$stmt, $sql] = $this->getDBStatement(); $count = 0; $totalCount = count($this->geoItems->items); - $progressBar = new \Symfony\Component\Console\Helper\ProgressBar($this->output, 100); + + $progressBar = new ProgressBar($this->output, 100); + foreach ($this->geoItems->items as $item) { - if ( $stmt->execute([ - ':id' => $item->getId(), - ':parent_id' => $item->parentId, - ':left' => $item->left, - ':right' => $item->right, - ':depth' => $item->depth, - ':name' => substr($item->data[2],0,40), - ':alternames' => $item->data[3], - ':country' => $item->data[8], - ':level' => $item->data[7], - ':population' => $item->data[14], - ':lat' => $item->data[4], - ':long' => $item->data[5] - ]) === false){ - //Before throwing enabling key checks - \DB::statement('SET FOREIGN_KEY_CHECKS=1;'); - $this->info('Relation checks enabled'); - throw new Exception("Error in SQL : '$sql'\n".PDO::errorInfo(), 1); + $params = [ + ':id' => $item->getId(), + ':parent_id' => $item->parentId, + ':left' => $item->left, + ':right' => $item->right, + ':depth' => $item->depth, + ':name' => substr($item->data[2], 0, 40), + ':alternames' => $item->data[3], + ':country' => $item->data[8], + ':a1code' => $item->data[10], + ':level' => $item->data[7], + ':population' => $item->data[14], + ':lat' => $item->data[4], + ':long' => $item->data[5], + ':timezone' => $item->data[17], + ]; + + if ($stmt->execute($params) === false) { + $error = "Error in SQL : '$sql'\n" . PDO::errorInfo() . "\nParams: \n$params"; + throw new Exception($error, 1); } - $progress = $count++/$totalCount*100; + + $progress = $count++ / $totalCount * 100; $progressBar->setProgress($progress); } - $progressBar->finish(); - - //Lets get back FOREIGN_KEY_CHECKS to laravel - \DB::statement('SET FOREIGN_KEY_CHECKS=1;'); - $this->info('Relation checks enabled'); - - $this->info(" Done"); - $time_elapsed_secs = microtime(true) - $start; - $this->info("Timing: $time_elapsed_secs sec"); + $progressBar->finish(); } } diff --git a/src/commands/seedJsonFile.php b/src/commands/seedJsonFile.php index 6e05b29..2deb8e4 100644 --- a/src/commands/seedJsonFile.php +++ b/src/commands/seedJsonFile.php @@ -1,58 +1,66 @@ -pdo = \DB::connection()->getPdo(\PDO::FETCH_ASSOC); - if (!\Schema::hasTable('geo')) + $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); + if (! Schema::hasTable('geo')) { return; + } $this->geoItems = new geoCollection(); } - public function handle() { + public function handle() + { $start = microtime(true); $filename = $this->argument('file'); - if(empty($filename)){ - $this->info("Available json files:"); - $this->info("---------------------"); - $files = array_diff(scandir(storage_path("geo")), ['.','..']); - foreach ($files as $file) - if(strpos($file, '.json')!==false) - $this->comment(' '.substr($file, 0, strpos($file, '.json'))); - $this->info("---------------------"); - $filename = $this->ask('Choose File to restore:'); + if (empty($filename)) { + $this->info('Available json files:'); + $this->info('---------------------'); + $files = array_diff(scandir(storage_path('geo')), ['.', '..']); + foreach ($files as $file) { + if (strpos($file, '.json') !== false) { + $this->comment(' ' . substr($file, 0, strpos($file, '.json'))); + } + } + $this->info('---------------------'); + $filename = $this->ask('Choose File to restore:'); } $filename = storage_path("geo/{$filename}.json"); $this->info("Parsing file: $filename"); $data = json_decode(file_get_contents($filename), true); - if($data === null){ - $this->error("Error decoding json file. Check for syntax errors."); + if ($data === null) { + $this->error('Error decoding json file. Check for syntax errors.'); exit(); } - $progressBar = new \Symfony\Component\Console\Helper\ProgressBar($this->output, count($data)); + $progressBar = new ProgressBar($this->output, count($data)); $count = 0; $rebuildTree = false; foreach ($data as $item) { - if ( isset($item['id']) && ($geo = Geo::find($item['id'])) ) + if (isset($item['id']) && ($geo = Geo::find($item['id']))) { $geo->update($item); - else { + } else { $item = array_merge([ 'alternames' => [], 'country' => '', @@ -69,8 +77,8 @@ public function handle() { $progressBar->finish(); $this->info(" Finished Processing $count items"); - if($rebuildTree){ - $this->info("Rebuilding Tree in DB"); + if ($rebuildTree) { + $this->info('Rebuilding Tree in DB'); Geo::rebuildTree($this->output); } diff --git a/src/commands/truncTable.php b/src/commands/truncTable.php index 1d3fac9..49132cd 100644 --- a/src/commands/truncTable.php +++ b/src/commands/truncTable.php @@ -1,6 +1,9 @@ -info('Relation checks disabled'); - \DB::table('geo')->truncate(); + DB::table('geo')->truncate(); $this->info('Table "geo" is empty now.'); } } diff --git a/src/dbTree/EloquentTreeItem.php b/src/dbTree/EloquentTreeItem.php index 2513dba..b69ee1f 100644 --- a/src/dbTree/EloquentTreeItem.php +++ b/src/dbTree/EloquentTreeItem.php @@ -1,88 +1,95 @@ -writeln('- '.$msg.''); - } + protected $parent = null; + protected $children = []; - public static function rebuildTree($output = null){ - - // Create associative array of all elements - self::print('Create associative array',$output); - foreach (self::all() as $item) { - self::$items[$item->id] = $item; - }; - - // Fill parent/children attributes - self::print('Create parent/children relations',$output); - foreach (self::$items as $item) { - if($item->parent_id){ - $item->parent = self::getItem($item->parent_id); - $item->parent->addChild($item); - } - } + public static function print($msg, $output) + { + if ($output) { + $output->writeln('- ' . $msg . ''); + } + } - // Build Tree for each Country (root) item - self::print('Build Tree',$output); - $count = 1; - foreach (self::$items as $item) { - if($item->level == GEO::LEVEL_COUNTRY){ - $count = self::buildTree($item, $count); - // $item->printTree(); - } - } + public static function rebuildTree($output = null, bool $printTree = false) + { + // Create associative array of all elements + self::print('Create associative array', $output); + foreach (self::all() as $item) { + self::$items[$item->id] = $item; + }; + + // Fill parent/children attributes + self::print('Create parent/children relations', $output); + foreach (self::$items as $item) { + if ($item->parent_id) { + $item->parent = self::getItem($item->parent_id); + $item->parent->addChild($item); + } + } - // Save in DB - self::print('Save in DB',$output); - foreach (self::$items as $item) { - $item->save(); - } + // Build Tree for each Country (root) item + self::print('Build Tree', $output); + $count = 1; + foreach (self::$items as $item) { + if ($item->level == GEO::LEVEL_COUNTRY) { + $count = self::buildTree($item, $count); + if ($printTree) { + $item->printTree(); + } + } + } + // Save in DB + self::print('Save in DB', $output); + foreach (self::$items as $item) { + $item->save(); + } } // Get item by id - private static function getItem($id){ - if(!isset(self::$items[$id])) - throw new \Exception("Item $id not found"); - - return self::$items[$id]; + private static function getItem($id) + { + if (! isset(self::$items[$id])) { + throw new \Exception("Item $id not found"); + } + return self::$items[$id]; } // Add $item as a child - private function addChild($item){ - $this->children[] = $item; + private function addChild($item) + { + $this->children[] = $item; } - private static function buildTree($item, $count = 1, $depth = 0){ - $item->left=$count++; - $item->depth=$depth; + private static function buildTree($item, $count = 1, $depth = 0) + { + $item->left = $count++; + $item->depth = $depth; foreach ($item->children as $child) { - $count = $item->buildTree($child, $count, $depth+1); + $count = $item->buildTree($child, $count, $depth + 1); } - $item->right=$count++; + $item->right = $count++; return $count; } - - public function printTree(){ - $levelStr= str_repeat('-', $this->depth); - echo(sprintf("%s %s [%d,%d]\n", $levelStr, $this->name,$this->left,$this->right)); - foreach ($this->children as $child) + public function printTree() + { + $levelStr = str_repeat('-', $this->depth); + echo(sprintf("%s %s [%d,%d]\n", $levelStr, $this->name, $this->left, $this->right)); + foreach ($this->children as $child) { $child->printTree(); + } } - - -} \ No newline at end of file +} diff --git a/src/migrations/2017_02_13_090952_geo.php b/src/migrations/2017_02_13_090952_geo.php index 07d8763..9290783 100644 --- a/src/migrations/2017_02_13_090952_geo.php +++ b/src/migrations/2017_02_13_090952_geo.php @@ -1,8 +1,8 @@ increments('id'); $table->integer('parent_id')->nullable(); $table->integer('left')->nullable(); $table->integer('right')->nullable(); $table->integer('depth')->default(0); - // $table->integer('geoid'); $table->char('name', 60); $table->text('alternames'); $table->char('country', 2); + $table->string('a1code', 25); $table->char('level', 10); $table->bigInteger('population'); - $table->decimal('lat',9,6); - $table->decimal('long',9,6); + $table->decimal('lat', 9, 6); + $table->decimal('long', 9, 6); + $table->char('timezone', 30); }); } - } /** @@ -37,7 +38,8 @@ public function up() { * * @return void */ - public function down() { + public function down() + { Schema::drop('geo'); } } diff --git a/src/routes.php b/src/routes.php index 1b4b8b2..14c48ed 100644 --- a/src/routes.php +++ b/src/routes.php @@ -1,11 +1,13 @@ 'api/geo', 'middleware' => 'api'], function(){ - Route::get('search/{name}/{parent_id?}', 'Igaster\LaravelCities\GeoController@search'); - Route::get('item/{id}', 'Igaster\LaravelCities\GeoController@item'); - Route::get('children/{id}', 'Igaster\LaravelCities\GeoController@children'); - Route::get('parent/{id}', 'Igaster\LaravelCities\GeoController@parent'); - Route::get('country/{code}', 'Igaster\LaravelCities\GeoController@country'); - Route::get('countries', 'Igaster\LaravelCities\GeoController@countries'); +Route::group(['prefix' => 'geo/'], function() { + Route::get('search/{name}/{parent_id?}', '\Igaster\LaravelCities\GeoController@search'); + Route::get('item/{id}', '\Igaster\LaravelCities\GeoController@item'); + Route::get('children/{id}', '\Igaster\LaravelCities\GeoController@children'); + Route::get('parent/{id}', '\Igaster\LaravelCities\GeoController@parent'); + Route::get('country/{code}', '\Igaster\LaravelCities\GeoController@country'); + Route::get('countries', '\Igaster\LaravelCities\GeoController@countries'); + Route::get('ancestors/{id}','\Igaster\LaravelCities\GeoController@ancestors'); + Route::get('breadcrumbs/{id}','\Igaster\LaravelCities\GeoController@breadcrumbs'); }); diff --git a/src/vue/geo-select.vue b/src/vue/geo-select.vue index b73438b..0e204b3 100644 --- a/src/vue/geo-select.vue +++ b/src/vue/geo-select.vue @@ -1,4 +1,4 @@ - \ No newline at end of file + } + From 98f044ae24fa446a763530613cbffc3e1414f726 Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Mon, 13 Jan 2020 20:06:23 +0530 Subject: [PATCH 2/9] added new option chunk to pass how many records to be processed at once to give flexibility to import large no of records with low memory footprints --- README.md | 7 ++++ src/Geo.php | 4 ++ src/commands/BuildPplTree.php | 5 ++- src/commands/seedGeoFile.php | 72 ++++++++++++++++++++++++++--------- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1d7010c..cf80dce 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,13 @@ artisan migrate artisan geo:seed ``` +You can also pass `--chunk` argument to specify how much chunk you want to process at once suppose you want `3000` records to be processed at once you can pass. +This gives flexibility to make the import with low memory footprints +``` +artisan geo:seed --chunk=3000 +``` +by default it is `1000` + Note: If you don't want all the countries, you can download only country specific files (eg US.txt) and import each one of them with: ``` diff --git a/src/Geo.php b/src/Geo.php index e45e997..678233d 100644 --- a/src/Geo.php +++ b/src/Geo.php @@ -12,6 +12,10 @@ class Geo extends EloquentTreeItem const LEVEL_COUNTRY = 'PCLI'; const LEVEL_CAPITAL = 'PPLC'; + + const LEVEL_PPL = 'PPL'; + // a populated city, town, village, or other agglomeration of buildings where people live and work + const LEVEL_1 = 'ADM1'; const LEVEL_2 = 'ADM2'; const LEVEL_3 = 'ADM3'; diff --git a/src/commands/BuildPplTree.php b/src/commands/BuildPplTree.php index 8c02692..0b40df0 100644 --- a/src/commands/BuildPplTree.php +++ b/src/commands/BuildPplTree.php @@ -4,6 +4,7 @@ use Illuminate\Console\Command; use Illuminate\Support\Arr; +use Illuminate\Support\Facades\DB; class BuildPplTree extends Command { @@ -43,7 +44,7 @@ private function downloadAdmin1CodesASCIIIfNotExists() private function determinePplParentId($country) { - $geos = \DB::table('geo') + $geos = DB::table('geo') ->select(['id', 'name', 'country', 'a1code', 'level']) ->where('country', $country) ->whereRaw('parent_id IS NULL') @@ -60,7 +61,7 @@ private function determinePplParentId($country) continue; } - \DB::table('geo')->where('id', $geo->id)->update(['parent_id' => $parentId]); + DB::table('geo')->where('id', $geo->id)->update(['parent_id' => $parentId]); } } diff --git a/src/commands/seedGeoFile.php b/src/commands/seedGeoFile.php index f66393e..b16fb48 100644 --- a/src/commands/seedGeoFile.php +++ b/src/commands/seedGeoFile.php @@ -13,12 +13,18 @@ class seedGeoFile extends Command { - protected $signature = 'geo:seed {country?} {--append}'; + protected $signature = 'geo:seed {country?} {--append} {--chunk=1000}'; protected $description = 'Load + Parse + Save to DB a geodata file.'; private $pdo; private $driver; + private $geoItems; + + private $batch = 0; + + private $chunkSize = 1000; + public function __construct() { parent::__construct(); @@ -31,6 +37,8 @@ public function __construct() if (! Schema::hasTable('geo')) { return; } + + $this->geoItems = new geoCollection(); } public function sql($sql) @@ -82,7 +90,7 @@ protected function getColumnsAsStringDelimated($delimeter = '"', bool $onlyPrefi $modifiedColumns = []; - foreach($columns as $column) { + foreach ($columns as $column) { $modifiedColumns[] = $delimeter . $column . (($onlyPrefix) ? '' : $delimeter); } @@ -91,7 +99,6 @@ protected function getColumnsAsStringDelimated($delimeter = '"', bool $onlyPrefi public function getDBStatement() : array { - $sql = "INSERT INTO {$this->getFullyQualifiedTableName()} ( {$this->getColumnsAsStringDelimated()} ) VALUES ( {$this->getColumnsAsStringDelimated(':', true)} )"; if ($this->driver == 'mysql') { @@ -101,7 +108,7 @@ public function getDBStatement() : array return [$this->pdo->prepare($sql), $sql]; } - public function readFile(string $fileName) + public function readFile(string $fileName, string $country = null) { $this->info("Reading File '$fileName'"); $filesize = filesize($fileName); @@ -132,7 +139,7 @@ public function readFile(string $fileName) case 'ADM3': // 8 sec case 'PPLA': // областные центры case 'PPLA2': // Корсунь - //case 'PPL': // Яблунівка + case 'PPL': // a city, town, village, or other agglomeration of buildings where people live and work // 185 sec $this->geoItems->add(new geoItem($line, $this->geoItems)); $count++; @@ -140,6 +147,10 @@ public function readFile(string $fileName) } $progress = ftell($handle) / $filesize * 100; $progressBar->setProgress($progress); + + if (count($this->geoItems->items) >= $this->chunkSize) { + $this->processItems($country); + } } $progressBar->finish(); @@ -147,26 +158,37 @@ public function readFile(string $fileName) $this->info(" Finished Reading File. $count items loaded"); } - public function handle() + public function processItems($country) { - $this->geoItems = new geoCollection(); + // Read hierarchy + $this->readHierarcy($country); + + // Build Tree + $this->buildTree(); + + // write to persistent storage + $this->writeToDb(); + + //reset the chunk + $this->geoItems->reset(); + + $this->info(PHP_EOL . 'Processed Batch ' . $this->batch); + $this->batch++; + } + public function handle() + { $start = microtime(true); $country = strtoupper($this->argument('country')); $sourceName = $country ? $country : 'allCountries'; $fileName = storage_path("geo/{$sourceName}.txt"); $isAppend = $this->option('append'); - $this->info("Start seeding for $country"); + $this->chunkSize = $this->option('chunk'); - // Read Raw file - $this->readFile($fileName); - - // Read hierarchy - $this->readHierarcy($country); + $this->info("Start seeding for $sourceName"); - // Build Tree - $this->buildTree(); + DB::beginTransaction(); // Clear Table if (! $isAppend) { @@ -174,13 +196,25 @@ public function handle() DB::table('geo')->truncate(); } - // Store Tree in DB - $this->writeToDb(); + // Read Raw file + $this->readFile($fileName, $country); + + // Read hierarchy + //$this->readHierarcy($country); + + // Build Tree + //$this->buildTree(); + // Store Tree in DB + //$this->writeToDb(); + //Lets get back FOREIGN_KEY_CHECKS to laravel DB::statement('SET FOREIGN_KEY_CHECKS=1;'); + $this->info(PHP_EOL . ' Relation checks enabled'); + DB::commit(); + $this->info(' Done'); $time_elapsed_secs = microtime(true) - $start; $this->info("Timing: $time_elapsed_secs sec"); @@ -233,14 +267,14 @@ public function buildTree() } $count++; - $this->info("+ Building Tree for Country: {$item->data[2]} #{$item->data[0]}"); + $this->info(PHP_EOL . "+ Building Tree for Country: {$item->data[2]} #{$item->data[0]}"); $maxBoundary = $this->buildDbTree($item, $maxBoundary, 0); // $this->printTree($item,$output); } } - $this->info("Finished: {$count} Countries imported. $countOrphan orphan items skiped"); + $this->info(PHP_EOL . "Finished: {$count} Countries imported. $countOrphan orphan items skiped"); } public function writeToDb() From 526ff082715e02ff88d5c690112944ef576603d3 Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Tue, 14 Jan 2020 18:37:20 +0530 Subject: [PATCH 3/9] updated down to dropIfExists for more fine grained apprach --- src/migrations/2017_02_13_090952_geo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/2017_02_13_090952_geo.php b/src/migrations/2017_02_13_090952_geo.php index 9290783..3e3c2ca 100644 --- a/src/migrations/2017_02_13_090952_geo.php +++ b/src/migrations/2017_02_13_090952_geo.php @@ -40,6 +40,6 @@ public function up() */ public function down() { - Schema::drop('geo'); + Schema::dropIfExists('geo'); } } From b4d39479e88252355c8e1037e97f27172a31c12f Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Tue, 14 Jan 2020 18:49:18 +0530 Subject: [PATCH 4/9] removed PPL while import as it gonna take long time removed PPL while import as it gonna take long time TODO make we may pass it as a flag from command --- src/commands/seedGeoFile.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/seedGeoFile.php b/src/commands/seedGeoFile.php index b16fb48..57eccea 100644 --- a/src/commands/seedGeoFile.php +++ b/src/commands/seedGeoFile.php @@ -139,7 +139,7 @@ public function readFile(string $fileName, string $country = null) case 'ADM3': // 8 sec case 'PPLA': // областные центры case 'PPLA2': // Корсунь - case 'PPL': // a city, town, village, or other agglomeration of buildings where people live and work + //case 'PPL': // a city, town, village, or other agglomeration of buildings where people live and work // 185 sec $this->geoItems->add(new geoItem($line, $this->geoItems)); $count++; From 04ff96fee0385c5f84310d1fc32832e7afce605a Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Tue, 14 Jan 2020 19:47:18 +0530 Subject: [PATCH 5/9] Update GeoServiceProvider.php --- src/GeoServiceProvider.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/GeoServiceProvider.php b/src/GeoServiceProvider.php index 9344e4b..bf3f941 100644 --- a/src/GeoServiceProvider.php +++ b/src/GeoServiceProvider.php @@ -8,9 +8,6 @@ class GeoServiceProvider extends ServiceProvider { public function boot() { - // Load migrations - $this->loadMigrationsFrom(__DIR__ . '/migrations'); - // Load Routes // $this->loadRoutesFrom(__DIR__.'/routes.php'); @@ -20,6 +17,10 @@ public function boot() // Register Commands if ($this->app->runningInConsole()) { + + // Load migrations + $this->loadMigrationsFrom(__DIR__ . '/migrations'); + $this->commands([ \Igaster\LaravelCities\commands\seedGeoFile::class, \Igaster\LaravelCities\commands\seedJsonFile::class, From 7c027c6d285e01daa4280dc5379af43333285a52 Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Tue, 14 Jan 2020 20:24:13 +0530 Subject: [PATCH 6/9] Update 2017_02_13_090952_geo.php --- src/migrations/2017_02_13_090952_geo.php | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/migrations/2017_02_13_090952_geo.php b/src/migrations/2017_02_13_090952_geo.php index 3e3c2ca..33e21b2 100644 --- a/src/migrations/2017_02_13_090952_geo.php +++ b/src/migrations/2017_02_13_090952_geo.php @@ -13,24 +13,22 @@ class Geo extends Migration */ public function up() { - if (! Schema::hasTable('geo')) { - Schema::create('geo', function (Blueprint $table) { - $table->increments('id'); - $table->integer('parent_id')->nullable(); - $table->integer('left')->nullable(); - $table->integer('right')->nullable(); - $table->integer('depth')->default(0); - $table->char('name', 60); - $table->text('alternames'); - $table->char('country', 2); - $table->string('a1code', 25); - $table->char('level', 10); - $table->bigInteger('population'); - $table->decimal('lat', 9, 6); - $table->decimal('long', 9, 6); - $table->char('timezone', 30); - }); - } + Schema::create('geo', function (Blueprint $table) { + $table->increments('id'); + $table->integer('parent_id')->nullable(); + $table->integer('left')->nullable(); + $table->integer('right')->nullable(); + $table->integer('depth')->default(0); + $table->char('name', 60); + $table->text('alternames'); + $table->char('country', 2); + $table->string('a1code', 25); + $table->char('level', 10); + $table->bigInteger('population'); + $table->decimal('lat', 9, 6); + $table->decimal('long', 9, 6); + $table->char('timezone', 30); + }); } /** From 93a481f578944b35225a9d725b56e8727027e576 Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Mon, 20 Jan 2020 11:38:11 +0530 Subject: [PATCH 7/9] Update seedGeoFile.php --- src/commands/seedGeoFile.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/commands/seedGeoFile.php b/src/commands/seedGeoFile.php index 57eccea..5c1c322 100644 --- a/src/commands/seedGeoFile.php +++ b/src/commands/seedGeoFile.php @@ -31,12 +31,6 @@ public function __construct() $connection = config('database.default'); $this->driver = strtolower(config("database.connections.{$connection}.driver")); - - $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); - - if (! Schema::hasTable('geo')) { - return; - } $this->geoItems = new geoCollection(); } @@ -178,6 +172,12 @@ public function processItems($country) public function handle() { + $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); + + if (! Schema::hasTable('geo')) { + return; + } + $start = microtime(true); $country = strtoupper($this->argument('country')); $sourceName = $country ? $country : 'allCountries'; From c823976827c782ecfc3fbc6c810c88c59237aedd Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Mon, 20 Jan 2020 11:38:38 +0530 Subject: [PATCH 8/9] Update seedJsonFile.php --- src/commands/seedJsonFile.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/commands/seedJsonFile.php b/src/commands/seedJsonFile.php index 2deb8e4..d2b09f6 100644 --- a/src/commands/seedJsonFile.php +++ b/src/commands/seedJsonFile.php @@ -18,16 +18,17 @@ class seedJsonFile extends Command public function __construct() { parent::__construct(); - $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); - if (! Schema::hasTable('geo')) { - return; - } $this->geoItems = new geoCollection(); } public function handle() { + $this->pdo = DB::connection()->getPdo(PDO::FETCH_ASSOC); + if (! Schema::hasTable('geo')) { + return; + } + $start = microtime(true); $filename = $this->argument('file'); From e9644e414afef811407fd2e940c5e40719e9bcaa Mon Sep 17 00:00:00 2001 From: Manash Sonowal Date: Mon, 20 Jan 2020 14:30:27 +0530 Subject: [PATCH 9/9] Update geoCollection.php --- src/commands/helpers/geoCollection.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/commands/helpers/geoCollection.php b/src/commands/helpers/geoCollection.php index f7d63c0..5c12246 100644 --- a/src/commands/helpers/geoCollection.php +++ b/src/commands/helpers/geoCollection.php @@ -35,4 +35,10 @@ public function findName($name) } return false; } + + public function reset() + { + $this->items = []; + return $this; + } }