diff --git a/.env.example b/.env.example index b677def..7ec19f2 100644 --- a/.env.example +++ b/.env.example @@ -47,3 +47,6 @@ PUSHER_APP_CLUSTER=mt1 MIX_PUSHER_APP_KEY="${PUSHER_APP_KEY}" MIX_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}" + +ECOWITT_ACCOUNT= +ECOWITT_PASSWORD= diff --git a/.gitignore b/.gitignore index 0ae59f0..ddb9e55 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /public/storage /storage/*.key /vendor +.idea .env .env.backup .phpunit.result.cache diff --git a/README.md b/README.md index 38b6823..59bc59b 100644 --- a/README.md +++ b/README.md @@ -4,108 +4,31 @@ ## General -In case your weather station send all data to ecowitt.net and you lost your weewx database, this huge cannonball will help you to reintegrate your data. +In case your weather station send all data to ecowitt.net and you lost your weewx database, +this huge cannonball will help you to reintegrate your data. The steps are easy. -### 1. Fetch +### 1. Convert -Get your data from 'undocumentend' ecowitt API. - -Use browser inspect tools to get session_id after login. You find this information within your cookies OR you can inspect XHR request. - -Replace SESSION_ID and DEVICE_ID in this bash script. - -*HINT*: I'm running Mac OS. You need to install `coreutils` before. **=>** `brew install coreutils` - -This dirty script can download your data on a daily base or per week. Adjust the script to your needs. -It will create JSON Files in your current directory. These JSON Files have to be converted in step 2. +This simple tool will login into your ecowitt.net account, fetch all available devices and download all available data +for the range between `startdate` and `enddate`. ```sh -#!/bin/bash -SESSION_ID="" # -DEVICE_ID="" # Fetch this ID from ecowitt URL. https://www.ecowitt.net/home/index?id= - -function week2date () { - local year=$1 - local week=$2 - local dayofweek=$3 - gdate -d "$year-01-01 +$(( $week * 7 + 1 - $(gdate -d "$year-01-04" +%u ) - 3 )) days -2 days + $dayofweek days" +"%Y-%m-%d" -} - -function getDateOfDay() { - local day=$1 - local year=$2 - gdate -d "$year-1-1 +$day days" +%F -} - -function loadByDate() { - local date=$1 - curl -s 'https://webapi.www.ecowitt.net/index/get_data' \ - -X 'POST' \ - -H 'Accept: application/json, text/plain, */*' \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -H 'Origin: https://www.ecowitt.net' \ - -H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \ - -H 'Content-Length: 93' \ - -H 'Accept-Language: en-us' \ - -H 'Host: webapi.www.ecowitt.net' \ - -H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Connection: keep-alive' \ - --data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${date}%2000%3A00&edate=${date}%2023%3A59&page=1" | gunzip > $date.json -} - - -function loadByWeek() { - local start=$1; - local end=$2; - local week=$3; - curl 'https://webapi.www.ecowitt.net/index/get_data' \ - -X 'POST' \ - -H 'Accept: application/json, text/plain, */*' \ - -H 'Content-Type: application/x-www-form-urlencoded' \ - -H 'Origin: https://www.ecowitt.net' \ - -H "Cookie: ousaite_session=${SESSION_ID}; ousaite_language=english; ousaite_loginstatus=1" \ - -H 'Content-Length: 93' \ - -H 'Accept-Language: en-us' \ - -H 'Host: webapi.www.ecowitt.net' \ - -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15' \ - -H "Referer: https://www.ecowitt.net/home/index?id=${DEVICE_ID}" \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Connection: keep-alive' \ - --data "device_id=${DEVICE_ID}&is_list=0&mode=0&sdate=${start}%2001%3A00&edate=${end}%2023%3A59&page=1" | gunzip > $week.json -} - -for day in {0..365}; do - DATE=$(getDateOfDay $day 2020); - echo "Fetching Date: $DATE"; - loadByDate $DATE; - sleep 1; -done - -for week in {1..53}; do - STARTDATE=$(week2date 2020 $week 1); - ENDDATE=$(week2date 2020 $week 7); - echo "Fetching Week $week ..." - - loadByWeek $STARTDATE $ENDDATE $week -done +php artisan ecowitt:export {startdate} {enddate} ``` -### 2. Convert - -Convert json files to importable format for wee_import using this cannonball tool: - +example ```sh -php artisan ecowitt:convert {path of json files} +php artisan ecowitt:export 2020-01-01 2020-12-31 ``` Details see source file: `/app/Console/Commands/EcowittConvertData.php` -This script will generate a CSV File which is compatible with wee_import config in step 3. The file will be generated in `/storage/app/weewx.csv`. +This script will generate a CSV File which is compatible with wee_import config in step 2. +The file will be generated in `/storage/app/ecowitt_.csv`. -### 3. import +### 2. import Import your data with wee_import. Sample config below @@ -309,4 +232,4 @@ source = CSV **DONE!** -regenerate / restart weewx daemon and check your data! \ No newline at end of file +regenerate / restart weewx daemon and check your data! diff --git a/app/Console/Commands/EcowittConvertData.php b/app/Console/Commands/EcowittConvertData.php index 5278155..0415be8 100644 --- a/app/Console/Commands/EcowittConvertData.php +++ b/app/Console/Commands/EcowittConvertData.php @@ -3,19 +3,27 @@ namespace App\Console\Commands; use App\Exports\WeewxExport; +use Carbon\Carbon; use Illuminate\Console\Command; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Http; use Maatwebsite\Excel\Facades\Excel; use SplFileInfo; class EcowittConvertData extends Command { + /** @var Carbon */ + protected $startDate; + + /** @var Carbon */ + protected $endDate; + /** * The name and signature of the console command. * * @var string */ - protected $signature = 'ecowitt:convert {path}'; + protected $signature = 'ecowitt:export {startDate} {endDate}'; /** * The console command description. @@ -41,19 +49,42 @@ class EcowittConvertData extends Command */ public function handle() { - $path = $this->argument('path'); - $files = collect(File::allFiles($path)); - $outputData = []; - $files->each(function(SplFileInfo $file) use (&$outputData) { - $ecowitt = json_decode(File::get($file->getPathname()),true); - if (empty(data_get($ecowitt, 'list.tempf'))) { - $this->comment("skip '{$file->getBasename()}'. next file"); - return; - } + $session_id = $this->getSessionId(); + $device_ids = $this->getDeviceIds($session_id); + + $this->startDate = Carbon::parse($this->argument('startDate'))->startOfDay() + ->format('Y-m-d H:i'); + + $this->startDate = Carbon::parse($this->argument('endDate'))->endOfDay() + ->format('Y-m-d H:i'); + + $device_ids->each(function ($deviceId) use ($session_id) { + + $startDate = $this->startDate; + $endDate = $this->endDate; + + $response = Http::withCookies( + [ + 'ousaite_session' => $session_id, + ], 'www.ecowitt.net') + ->asForm() + ->post('https://webapi.www.ecowitt.net/index/get_data', [ + 'device_id' => $deviceId, + 'is_list' => 0, + 'mode' => 0, + 'sdate' => $startDate, + 'edate' => $endDate, + 'page' => 1, + ]); + + $ecowitt = $response->json(); + + // declare output variable + $outputData = []; // Temperature in C $outdoorTemp = $this->getData($ecowitt, 'list.tempf.list.tempf'); - + // Feels Like in C $outdoorTempGust = $this->getData($ecowitt, 'list.tempf.list.sendible_temp'); @@ -74,7 +105,7 @@ class EcowittConvertData extends Command // uv $uvi = $this->getData($ecowitt, 'list.uv.list.uv'); - + // rainrate in mm/hr b $rainRateH = $this->getData($ecowitt, 'list.rain.list.rainratein'); @@ -96,7 +127,7 @@ class EcowittConvertData extends Command // pressure absolute in hPa $pressureAbs = $this->getData($ecowitt, 'list.pressure.list.baromabsin'); - foreach($outdoorTemp as $date => $temp) { + foreach ($outdoorTemp as $date => $temp) { $tmp = [ 'date_and_time' => $date, // %Y-%m-%d %H:%M:%S 'temp_out' => $temp, // degree @@ -121,14 +152,65 @@ class EcowittConvertData extends Command ]; $outputData[] = $tmp; } + + Excel::store( + new WeewxExport($outputData), + "ecowitt_{$deviceId}.csv", + null, + \Maatwebsite\Excel\Excel::CSV + ); }); - Excel::store(new WeewxExport($outputData), 'weewx.csv', null, \Maatwebsite\Excel\Excel::CSV); } - public function getData($stack, $key) { + protected function getData($stack, $key) + { return collect(data_get($stack, $key)) ->mapWithKeys(function ($value) { return [$value[0] => $value[1] ?: null]; }); } + + /** + * simulate login to ecowitt.net and store session_id from cookie + * + * @return mixed + */ + protected function getSessionId() + { + $response = Http::asForm()->post('https://webapi.www.ecowitt.net/user/site/login', [ + 'account' => env('ECOWITT_ACCOUNT'), + 'password' => env('ECOWITT_PASSWORD'), + 'authorize' => '', + ]); + + $loginData = $response->json(); + + $cookies = $response->cookies(); + + return $cookies->getCookieByName('ousaite_session')->getValue(); + } + + /** + * fetch all available device IDs + * @param $session_id + * @return \Illuminate\Support\Collection + */ + protected function getDeviceIds($session_id): \Illuminate\Support\Collection + { + $deviceResponse = Http::withCookies( + [ + 'ousaite_session' => $session_id, + ], 'www.ecowitt.net') + ->asForm() + ->post('https://webapi.www.ecowitt.net/index/get_devices', [ + 'uid' => '', + 'type' => 1 + ]); + + $devices = collect(data_get($deviceResponse->json(), 'device_list')); + + return $devices->map(function ($device) { + return data_get($device, 'id'); + }); + } } diff --git a/composer.json b/composer.json index 4ac5073..bea4e89 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,7 @@ "php": "^7.3|^8.0", "fideloper/proxy": "^4.4", "fruitcake/laravel-cors": "^2.0", - "guzzlehttp/guzzle": "^7.0.1", + "guzzlehttp/guzzle": "^7.2", "laravel/framework": "^8.12", "laravel/tinker": "^2.5", "maatwebsite/excel": "^3.1" diff --git a/composer.lock b/composer.lock index 8590fe2..5b1abe5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ad0023f01098d001a84f01dcfc5def40", + "content-hash": "36fa813c4a4a2cd2fcc15dfde8f14463", "packages": [ { "name": "asm89/stack-cors", diff --git a/routes/console.php b/routes/console.php index 82f99fd..c09da46 100644 --- a/routes/console.php +++ b/routes/console.php @@ -4,7 +4,7 @@ use Illuminate\Foundation\Inspiring; use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; - +use Illuminate\Support\Facades\Http; /* |-------------------------------------------------------------------------- | Console Routes