diff --git a/.github/workflows/github-actions-demo.yml b/.github/workflows/github-actions-demo.yml new file mode 100644 index 00000000..09c778fa --- /dev/null +++ b/.github/workflows/github-actions-demo.yml @@ -0,0 +1,17 @@ +# name: GitHub Actions Demo +# on: [push] +# jobs: +# Explore-GitHub-Actions: +# runs-on: ubuntu-latest +# steps: +# - run: echo "๐ŸŽ‰ The job was automatically triggered by a ${{ github.event_name }} event." +# - run: echo "๐Ÿง This job is now running on a ${{ runner.os }} server hosted by GitHub!" +# - run: echo "๐Ÿ”Ž The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." +# - name: Check out repository code +# uses: actions/checkout@v2 +# - run: echo "๐Ÿ’ก The ${{ github.repository }} repository has been cloned to the runner." +# - run: echo "๐Ÿ–ฅ๏ธ The workflow is now ready to test your code on the runner." +# - name: List files in the repository +# run: | +# ls ${{ github.workspace }} +# - run: echo "๐Ÿ This job's status is ${{ job.status }}." diff --git a/.github/workflows/markdown-lint.yml b/.github/workflows/markdown-lint.yml new file mode 100644 index 00000000..01f70ae8 --- /dev/null +++ b/.github/workflows/markdown-lint.yml @@ -0,0 +1,25 @@ +name: GitHub Actions Demo + +on: [push] + +jobs: + +# node-docker: +# runs-on: ubuntu-latest +# container: +# image: node:14.15.0-alpine3.12 +# steps: +# - name: Log the node version +# run: | +# node -v +# cat /etc/os-release + + lint-markdown: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: debug output + uses: docker://ghcr.io/igorshubovych/markdownlint-cli:latest + with: + args: ./notes/t*.md --disable MD013 MD025 MD010 diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..234a059d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.DS_Store +.obsidian/ diff --git a/README.md b/README.md index f630ab60..f85db27c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,18 @@ # notable - notes -my notes managed with [notable/notable](https://github.com/notable/notable) +> collection of markdown notes about tools used on the cli + +It's a bit of a combination of [command-not-found.com](https://command-not-found.com) and [tldr.sh](https://tldr.sh/) for me. +Most notes describe a cli program, a short description, how to install, a usage section and references to other tools. + +## prerequesites + +- [github.com/notable/notable](https://github.com/notable/notable) + +## usage + +- clone repo +- open repo dir using `notable` +- start taking notes + +Have fun ! diff --git a/attachments/docker-compose.terraform-run.yml b/attachments/docker-compose.terraform-run.yml new file mode 100644 index 00000000..e3bd96d7 --- /dev/null +++ b/attachments/docker-compose.terraform-run.yml @@ -0,0 +1,18 @@ +version: '2.4' + +services: + terraform: + build: + context: . + args: + TAG: 0.12.6 + image: docker-registry/terraform:0.12.6 + tty: true + stdin_open: true + environment: + TF_VAR_vsphere_server: ${vsphere_server} + TF_VAR_vsphere_user: ${vsphere_user} + TF_VAR_vsphere_password: ${vsphere_password} + volumes: + - ./:/opt/terraform/ + - /var/run/docker.sock:/var/run/docker.sock diff --git a/attachments/icon.png b/attachments/icon.png deleted file mode 100644 index 88599a52..00000000 Binary files a/attachments/icon.png and /dev/null differ diff --git a/attachments/icon_small.png b/attachments/icon_small.png deleted file mode 100644 index 71322820..00000000 Binary files a/attachments/icon_small.png and /dev/null differ diff --git a/notes/12 factor app.md b/notes/12 factor app.md new file mode 100644 index 00000000..a71a72c1 --- /dev/null +++ b/notes/12 factor app.md @@ -0,0 +1,111 @@ +--- +tags: [Notebooks] +title: 12 factor app +created: '2022-02-15T10:15:31.700Z' +modified: '2023-04-25T08:26:28.980Z' +--- + +# 12 factor app + +> methodology for building software-as-a-service apps + +## 12 factors + +- was developed by heroku +- is as set of best practices aswell as a mindset + +## I. Codebase + +> one codebase tracked in a vcs, many deploy + +- multiple developers push to single source base via git +- services that comprise one app should each have a seperate codebase + +## II. Dependencies + +> explicitly declare and isolate dependencies + +- a twelve factor app never relies on implicit existence of system wite packages +- declare dependecies explicitly e.g. `flask==2.0.0` in `requirements.txt` +- isolate environment using virtual environments like [[virtualenv]] or [[docker]] + +## III. Config + +> store config in the environment + +- store config values in environment variables +- seperate set of environment variable per stage + +## IV. Backing Services + +> treat backing services as attached resources + +- attached resoruce meaning it should be no problem switching to a differen redis instance + +## V. Build, Release, Run + +> strictly seperate build and run stages + +- strict seperation between `build`, `release`and `run` stages +1. build: generate executable -> e.g `docker build` +2. release: executable + config = `release object`, should have release id, tag +3. run: + +## VI. Processes + +> execute app as one or more stateless processes + +- processes are stateless and share-nothing +- sticky sessions are a violation of twelve-factor and should never be used or relied upon if data is stored locally on the process (data loss!) +- use `IV. Backing Services` like redis or a db to share data between processes + +## VII. Port Binding + +> export services via port binding + +- twelve-factor app is completely self-contained and does not rely on specific server to function +- [[osi model]] [[tcp-ip model]] + +## VIII. Concurrency + +> scale out via process model + +- to use horizontal scaling the app must be independant and stateless +- processes are first class citizen -> app should scale horizontally not vertically -> `VI. Processes` + +## IV. Disposability + +> maximize robustness with fast startup and graceful shutdown + +- app's processes are disposable and should be started or stopped at a moments notice +- app's processes should shutdown gracefully when receiving a `SIGTERM` [[signal]] from process manager, avoiding unexpected dataloss + +## X. Dev/prod parity + +> keep development, staging and production as similar as possible + +- designed for Continous Deployment by keepking gap between dev and prod small +- developer resists the urge to use different backing services betweetn dev and prod + +## XI. Logs + +> treat logs as event streams + +- twelve factor app never concerns itself with routing or storage of it output stream +- store logs in centralized location in structured format +- don't store logs in local fs -> `IV. Disposability` +- ouput logs to `STDOUT` which can be transported by agent (fluentd) to centralized logging system (ELK-Stack, Splunk) + +## XII. Admin Processes + +> rund admin/management tasks as one-off processes + +- administrative tasks should be kept seperate from app process +- database migration or restart should be run as seperate process e.g. start coontainer in parallel to run migration + +## see also + +- [[devops]] +- [[unix socket]] +- [12factor.net](https://12factor.net/) +- [[iac]] diff --git a/notes/7z.md b/notes/7z.md new file mode 100644 index 00000000..e4661fb2 --- /dev/null +++ b/notes/7z.md @@ -0,0 +1,25 @@ +--- +tags: [linux, macos] +title: 7z +created: '2023-01-27T09:15:35.429Z' +modified: '2023-03-25T12:25:14.303Z' +--- + +# 7z + +## install + +```sh +brew install p7zip +``` + +## usage + +```sh +7z x FILE.rar +``` + +## see also + +- [[rar]] +- [[zip]] diff --git a/notes/BIND.md b/notes/BIND.md new file mode 100644 index 00000000..87db8bf4 --- /dev/null +++ b/notes/BIND.md @@ -0,0 +1,40 @@ +--- +tags: [dns] +title: BIND +created: '2019-07-30T06:19:49.027Z' +modified: '2023-03-20T08:58:45.389Z' +--- + +# BIND + +> `berkeley internet name domain` - implements the DNS protocols and a software distribution which contains all of the software necessary for `asking` and `answering` name service questions + +### The BIND software distribution has three parts + +1. domain name resolver + resolver is a program that resolves questions about names by sending those questions to appropriate servers and responding appropriately to the servers + stub resolver library +2. domain name authority server - authoritative DNS server answers requests from resolvers +3. tools - diagnostic and operational tools such as the `dig` + +## install + +```sh +yum install bind bind-utils +``` + +## usage + +`BIND`โ€™s process is known as [[named]] + +```sh +/var/lib/named/slave/domain.net.zone +``` + +## see also + +- [[dig]] +- [[named]] +- [[dnsmasq]] +- [[rndc]] +- [Domain name resolution - ArchWiki](https://wiki.archlinux.org/index.php/resolv.conf) diff --git a/notes/acid crud.md b/notes/acid crud.md new file mode 100644 index 00000000..b3814395 --- /dev/null +++ b/notes/acid crud.md @@ -0,0 +1,68 @@ +--- +tags: [Notebooks] +title: acid crud +created: '2019-08-18T16:14:42.254Z' +modified: '2022-02-02T08:49:37.072Z' +--- + +# acid crud + +## acid + +> set of properties that guarante reliable database transactions + +``` +A C I D +| | | โ””โ”€ Durability +| | โ””โ”€โ”€โ”€ Isolation +| โ””โ”€โ”€โ”€โ”€โ”€ Consistency +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Atomaticity +``` + +#### atomicity + +> Transactions are composed of multiple statements. Atomicity guarantees that each transaction is treated as a single "unit", which either succeeds completely, or fails completely: if any of the statements constituting a transaction fails to complete, the entire transaction fails and the database is left unchanged. + +#### consistency + +> ensures that a transaction can only bring the database from one valid state to another - any data written to the database must be valid according to all defined rules, including constraints, cascades, triggers, and any combination thereof. This prevents database corruption by an illegal transaction, but does not guarantee that a transaction is correct. + +#### isolation + +> Transactions are often executed concurrently (e.g., multiple transactions reading and writing to a table at the same time). Isolation ensures that concurrent execution of transactions leaves the database in the same state that would have been obtained if the transactions were executed sequentially. + +#### durability + +> guarantees that once a transaction has been committed, it will remain committed even in the case of a system failure (power outage or crash). This usually means that completed transactions are recorded in non-volatile memory + +> ensure that the data is actually written on disk physically, preventing any loss of data in case of a sudden power outage + +On POSIX systems, durability is achieved through `sync` operations: `fsync`, `fdatasync`, `aio_fsync` +`fsync` is intended to force a physical write of data from the buffer cache, and to assure that after a system crash/failure that all data up to the time of the `fsync` call is recorded on the disk. +`fdatasync()` does not necessarily update the meta-data associated with a file โ€“ such as the `last modified` date โ€“ but only the file data + +## crud + +> 4 basic functions of persistent storage + +``` +C R U D +| | | โ””โ”€ Delete +| | โ””โ”€โ”€โ”€ Update +| โ””โ”€โ”€โ”€โ”€โ”€ Read +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Ceate +``` + + +| CRUD-Operation | SQL-92 | HTTP/REST | +|-- |-- |-- | +| Create | `INSERT` | `PUT` or `POST` | +| Read (Retrieve) | `SELECT` | `GET` | +| Update | `UPDATE` | `PATCH` or `PUT` | +| Delete (Destroy) | `DELETE` | `DELETE` | + +## see also + +- [[rest api design]] +- [blog.httrack.com/../everything-you-always-wanted-to-know-about-fsync](http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/) +- [lwn.net/postgresqls-fsync-surprise](https://lwn.net/Articles/752063/) diff --git a/notes/add-apt-repository.md b/notes/add-apt-repository.md new file mode 100644 index 00000000..20b099db --- /dev/null +++ b/notes/add-apt-repository.md @@ -0,0 +1,39 @@ +--- +tags: [linux] +title: add-apt-repository +created: '2022-02-02T09:05:29.982Z' +modified: '2023-03-23T10:17:46.175Z' +--- + +# add-apt-repository + +> script which adds an external [[apt]] repository to either `/etc/apt/sources.list` +> or a file in `/etc/apt/sources.list.d/` +> or removes an already existing repository + +## option + +```sh +-h, --help # show help message and exit +-m, --massive-debug # print a lot of debug information to the command line +-r, --remove # remove the specified repository +-y, --yes # assume yes to all queries +-k, --keyserver # use custom keyserver instead of the default +-s, --enable-source # allow downloading of the source packages from the repository +``` + +## usage + +```sh +add-apt-repository "deb https://download.srcclr.com/ubuntu stable/" +add-apt-repository "ppa:ian-berke/ppa-drawers" # ppa + +add-apt-repository -r "deb https://download.srcclr.com/ubuntu stable/" +add-apt-repository -r "ppa:ian-berke/ppa-drawers" +``` + +## see also + +- [[apt-key]] +- [[apt]] +- [[apt-get]] diff --git a/notes/adduser.md b/notes/adduser.md new file mode 100644 index 00000000..4cb2cd8b --- /dev/null +++ b/notes/adduser.md @@ -0,0 +1,23 @@ +--- +tags: [linux] +title: adduser +created: '2020-03-23T12:12:51.817Z' +modified: '2022-02-02T08:49:45.771Z' +--- + +# adduser + +> uses `useradd` as backend and adds user with full profile and info (pass, quota, permission, etc.) + +## usage + +```sh +adduser USER + +adduser --home USER_HOME --system --group USER +``` + +## see also + +- [[useradd]] +- [[usermod]] diff --git a/notes/airport.md b/notes/airport.md new file mode 100644 index 00000000..12f12cc3 --- /dev/null +++ b/notes/airport.md @@ -0,0 +1,28 @@ +--- +tags: [macos, network] +title: airport +created: '2019-07-30T06:19:49.196Z' +modified: '2022-02-02T08:49:57.619Z' +--- + +# airport + +> get information for 802.11 interface + +## usage + +```sh +ln -s /System/Library/PrivateFrameworks/Apple80211.framework/Versions/Current/Resources/airport /usr/local/bin/airport + +airport -s # Scan for Wi-Fi networks + +airport -c CHANNEL # Change channel + +airport -z # Disconnect + +airport -I # Get current connection info +``` + +## see also + +- [[networksetup]] diff --git a/notes/alacritty.md b/notes/alacritty.md new file mode 100644 index 00000000..ffb950f2 --- /dev/null +++ b/notes/alacritty.md @@ -0,0 +1,55 @@ +--- +tags: [linux, macos, rust] +title: alacritty +created: '2023-03-07T07:58:26.959Z' +modified: '2023-05-10T14:13:42.703Z' +--- + +# alacritty + +> terminal emulator, sensible defaults, extensive configuration + +## install + +```sh +brew install alacritty + +cargo install alacritty +``` + +## option + +```sh + --class INSTANCE # Defines window class/app_id on X11/Wayland [default: Alacritty] + --config-file FILE # alternative configuration file [default: $HOME/.config/alacritty/alacritty.yml] +-e, --command CMD... # command and args to execute (must be last argument) + --embed EMBED # X11 window ID to embed Alacritty within (decimal or hexadecimal with "0x" prefix) +-h, --help # Print help information + --hold # Remain open after child process exit +-o, --option OPT... # override configuration file options [example: cursor.style=Beam] + --print-events # print all events to stdout +-q # reduces the level of verbosity (min level is -qq) + --ref-test # generates ref test + --socket SOCKET # path for IPC socket creation +-t, --title TITLE # defines the window title [default: Alacritty] +-v # increases the level of verbosity (the max level is -vvv) +-V, --version # print version information + --working-directory DIR # start shell in specified working directory +``` + +## usage + +```sh +alacrity -v # start session verbose + +alacrity help # print help +alacrity msg # send message to the Alacritty socket +``` + +## see also + +- [[brew]] +- [[cargo]] +- [[tmux]] +- [[bash]] +- [[nerdfonts]] diff --git a/notes/amazon-linux-extras.md b/notes/amazon-linux-extras.md new file mode 100644 index 00000000..b0a61955 --- /dev/null +++ b/notes/amazon-linux-extras.md @@ -0,0 +1,32 @@ +--- +tags: [linux] +title: amazon-linux-extras +created: '2022-01-24T10:53:34.798Z' +modified: '2023-03-22T10:21:07.164Z' +--- + +# amazon-linux-extras + +> manage specific fresh software + +## install + +```sh +yum install -y amazon-linux-extras +``` + +## usage + +```sh +amazon-linux-extras # list + +amazon-linux-extras enable php7.2 # enable desired topic + +amazon-linux-extras install -y postgresql12 # +``` + +## see also + +- [[aws]] +- [[psql]] +- [[yum]] diff --git a/notes/ansible-playbook.md b/notes/ansible-playbook.md new file mode 100644 index 00000000..62cbc89f --- /dev/null +++ b/notes/ansible-playbook.md @@ -0,0 +1,26 @@ +--- +tags: [iac] +title: ansible-playbook +created: '2019-11-28T11:53:47.807Z' +modified: '2022-02-02T08:50:15.733Z' +--- + +# ansible-playbook + +> runs ansible playbooks, executing the defined tasks on the targeted hosts. + +## usage + +```sh +ansible-playbook --syntax-check playbook.yml + +ansible-playbook --list-hosts playbook.yml + +ansible-playbook --list-tasks playbook.yml + +ansible-playbook -i hosts ansible-bootstrap-ubuntu-16.04.yml # bootstraps machine with python2 +``` + +## see also + +- [[ansible]] diff --git a/notes/ansible.md b/notes/ansible.md index 0fb7e051..6bee2227 100644 --- a/notes/ansible.md +++ b/notes/ansible.md @@ -2,29 +2,24 @@ tags: [iac] title: ansible created: '2019-07-30T06:19:27.514Z' -modified: '2019-07-30T09:06:44.963Z' +modified: '2022-02-02T08:50:08.080Z' --- # ansible -## ansible adhoc +> software provisioning, configuration management, and application-deployment tool enabling infrastructure as code + +## usage ```sh -ansible all -m ping -s +ansible all -m ping -s # adhoc command -ansible -i hosts all -m ping # uses local hosts file +ansible all -m shell -a 'cat /etc/*release' # adhoc command using shell string -ansibele all -m shell -a 'cat /etc/*release' +ansible -i hosts all -m ping # uses local hosts file ``` -## ansible-playbook - -```sh -ansible-playbook --syntax-check playbook.yml - -ansible-playbook --list-hosts playbook.yml +## see also -ansible-playbook --list-tasks playbook.yml - -ansible-playbook -i hosts ansible-bootstrap-ubuntu-16.04.yml # bootstraps machine with python2 -``` +- [[ansible-playbook]] +- [[ssh]] diff --git a/notes/ant.md b/notes/ant.md new file mode 100644 index 00000000..61f068a2 --- /dev/null +++ b/notes/ant.md @@ -0,0 +1,38 @@ +--- +tags: [buildsystem, java] +title: ant +created: '2019-08-20T07:46:08.815Z' +modified: '2022-02-02T08:50:25.068Z' +--- + +# ant + +> completely written in java - `ant` uses `xml` as build script + +## usage + +```sh +ant -version + +ant compile + +ant package + +ant hello # run target: hello +``` + +## target + +```xml + + + + Hello, World! + + +``` + +## see also + +- [[mvn]] +- [[java]] diff --git a/notes/apache httpd.md b/notes/apache httpd.md deleted file mode 100644 index d4bf0d1b..00000000 --- a/notes/apache httpd.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -tags: [cli, linux] -title: apache httpd -created: '2019-07-30T06:19:48.985Z' -modified: '2019-07-30T09:05:39.597Z' ---- - -# apache httpd - -> `A Patchy Server` - -```sh -# prerequesite -source /etc/apache2/envvars \ -apache2 -S # Show settings as parsed from the config file -``` - -```sh -a2enmode ssl # enable-modules - -# /server-status -a2enmod info.load -a2enmod info.conf # vim /etc/apach2/mods-available/info.conf => comment Require to access outside localhost -service apache2 restart - -a2ensite vhosts_nam.com.conf # enable site -``` - -## apachectl - -### test config -```sh -apachectl configtest - -/usr/local/apache/bin/apachectl configtest -``` -### list modules -```sh -apache2ctl -M # list loaded modules - -apachectl -t -D DUMP_MODULES # list loaded modules - -ls /etc/apache2/mods-enabled/ # list enabled modules -ls /etc/apache2/mods-available/ # list available modules -``` -[Generate Mozilla Security Recommended Web Server Configuration Files](https://mozilla.github.io/server-side-tls/ssl-config-generator/) diff --git a/notes/apache2.md b/notes/apache2.md new file mode 100644 index 00000000..c622f0bf --- /dev/null +++ b/notes/apache2.md @@ -0,0 +1,76 @@ +--- +tags: [linux] +title: apache2 +created: '2019-07-30T06:19:48.985Z' +modified: '2023-04-11T15:53:46.578Z' +--- + +# apache2 + +> `A Patchy Server` and package + +## install + +```sh +apt-get install apache2 +``` + +## usage + +```sh +# prerequesite +source /etc/apache2/envvars \ +apache2 -S # Show settings as parsed from the config file +apache2 -v + +service apache2 restart +``` + +## apachectl + +```sh + +apachectl configtest # test config + +/usr/local/apache/bin/apachectl configtest + + +apache2ctl -M # list loaded modules + +apachectl -t -D DUMP_MODULES # list loaded modules + +ls /etc/apache2/mods-enabled/ # list enabled modules +ls /etc/apache2/mods-available/ # list available modules +``` + +## a2enmode + +```sh +a2enmode ssl # enable-modules + +a2enmod info.load # /server-status + +a2enmod info.conf +# vim /etc/apach2/mods-available/info.conf => comment Require to access outside localhost +``` + +## a2ensite + +```sh +a2ensite vhosts_nam.com.conf # enable site / virtual host +``` + +```sh +-q, --quiet # don't show informative messages. +-m, --maintmode # enables the maintainer mode +-p, --purge # purge all traces of the module in the internal state data base +``` + +```sh +a2dissite 000-default +``` + +## see also + +- [Generate Mozilla Security Recommended Web Server Configuration Files](https://mozilla.github.io/server-side-tls/ssl-config-generator/) +- [manpages.debian.org/jessie/apache2/a2ensite.8.en.html](https://manpages.debian.org/jessie/apache2/a2ensite.8.en.html) diff --git a/notes/api consul.md b/notes/api consul.md new file mode 100644 index 00000000..facc43bf --- /dev/null +++ b/notes/api consul.md @@ -0,0 +1,53 @@ +--- +tags: [iac] +title: api consul +created: '2019-07-30T06:19:49.077Z' +modified: '2023-03-22T09:58:30.041Z' +--- + +# api consul + +> api can perform basic CRUD operations on nodes, services, checks, configuration etc. + +## usage + +```sh +GET /v1/status/leader + +# /agent endpoint used to interact with local consul-agent ! +GET /v1/agent/members + +PUT /v1/agent/service/register # add new service to local agent + +# /catalog endpoint register/deregister nodes, services, and checks in Consul +GET /v1/catalog/services + +GET /v1/catalog/services | jq -r 'to_entries[].key' # get only service names + +GET /v1/catalog/service/SERVICE_NAME | jq -r '.[] | "\(.Node) \(.ServicePort) \(.ServiceID)"' + +GET /v1/catalog/service/SERVICENAME | jq '.[].ServiceID' # get service id + +GET /v1/catalog/register + +PUT /v1/catalog/register +{ + "Datacenter": "dc1", + "Node": "NODE", + "Address": "1.2.3.4", + "ServiceName": "SERVICE_NAME" +} + +PUT /v1/catalog/deregister +{ + "Datacenter": "dc1", + "Node": "NODE", + "ServiceID": "NODE-vip-svc:k4dv467p..qw07xh:HOST:PORT" +} +``` + +## see also + +- [[consul]] +- [[vault]] +- [[curl]] diff --git a/notes/api design rest.md b/notes/api design rest.md new file mode 100644 index 00000000..240cdd42 --- /dev/null +++ b/notes/api design rest.md @@ -0,0 +1,276 @@ +--- +tags: [Notebooks] +title: api design rest +created: '2019-07-30T06:19:49.224Z' +modified: '2023-03-25T12:32:33.176Z' +--- + +# api design rest + +> from `RESTful API Design โ€“ OCTO Quick Reference Card` + +## general concepts + +## KISS + +Anyone should be able to use your API without having to refer to the documentation. +- Use standard, concrete and shared terms, not your specific business terms or acronyms. +- Never allow application developers to do things more than one way. +- Design your API for your clients + +(Application developers), not for your data. +- Target major uses cases first, deal with exceptions later + +```sh +GET /orders +GET /users +GET /products + +curl โ€“XPOST -H "Accept: application/json" \ + -H "Authorization: Bearer at-80003004-19a8-46a2-908e-33d4057128e7" \ + -d '{"state":"running"}' \ + https://api.fakecompany.com/v1/users/007/orders?client_id=API_KEY_003 +``` + +### Medium grained resources + +You should use medium grained, not fine nor coarse. Resources shouldnโ€™t be nested more than two level deep e.g. + +```json +GET /users/007 +{ + "id":"007", + "firstname":"James", + "name":"Bond", + "address":{ + "street":"H.Ferry Rd.", + "country": { "name":"London" } + } +} +``` + +consider the following five subdomains +Production `https://api.fakecompany.com` +Tests `https://api.sandbox.fakecompany.com` +Developer portal `https://developers.fakecompany.com` +Production `https://oauth2.fakecompany.com` +Tests `https://oauth2.sandbox.fakecompany.com` + +--- + +## URLs + +### Nouns + +You should use `nouns`, not `verbs` (vs SOAP-RPC). `GET /orders` not `/getAllOrders` + +### Plurals + +use plural nouns not singular nouns to manage two different types of resources : +- CollecTon resource : `/users ` +- Instance resource : `/users/007` + +You should remain consistent. `GET /users/007` not `GET /user/007` + +### Consistent case + +choose between `snake_case` or `camelCase` for attributes and parameters, but remain consistent +- `GET /orders?id_user=007` or `GET /orders?idUser=007` +- `POST/orders {"id_user":"007"}` or `POST/orders {"idUser":"007"}` + +if you have to use more than one word in URL, you should use spinal-case (some servers ignore case). `POST /specific-orders` + +### Versioning + +make versioning mandatory in the URL at the highest scope (major versions). You may support at most two versions at the same time +`GET /v1/orders` + +### Hierarchical structure + +leverage the hierarchical nature of the URL to imply structure (aggregatoon or composition) +e.g.: an order contains products. +`GET /orders/1234/products/1` + +## CRUD-like operations + +Use HTTP verbs for CRUD operations `Create/Read/Update/Delete` + +```sh +Verb Collection: /orders Instance: /orders/{id} + +GET Read a list orders. 200 OK Read detail of single order. 200 OK +POST Create a new order. 201 Created - +PUT - Full Update : 200 OK / Create a specific order: 201 Created +PATCH - Partial Update. 200 OK +DELETE - Delete order. 204 OK +``` + + +```sh +# POST is used to Create an instance of a collection. +# The ID isnโ€™t provided, and the new resource location is returned in the `Location` Header. +POST /orders {"state":"running", "id_user":"007"} +201 Created +Location: https://api.fakecompany.com/orders/1234 + +# But remember that, if the ID is specified by the client, + +# `PUT` is used for Create. +PUT /orders/1234 +201 Created + +#`PUT` is used for Update to perform a full replacement. +PUT /orders/1234 {"state":"paid", "id_user":"007"} +200 Ok + + +#`PATCH` is commonly used for partial Update. +PATCH /orders/1234 {"state":"paid"} +200 Ok + +#`GET` is used to Read a collection. +GET /orders +200 Ok +[ + {"id":"1234", "state":"paid"} + {"id":"5678", "state":"running"} +] + + +# GET is used to Read an instance. +GET /orders/1234 +200 Ok +{"id":"1234", "state":"paid"} +``` + +--- + +## Query strings + +### Search + +You may use the โ€œGoogle wayโ€ to perform a global search on multiple resources. +`GET /search?q=running+paid` + +### Filters + +You should use `?` to filter resources +`GET /orders?state=payed&id_user=007` +or (multiple URIs may refer to the same resource) +`GET /users/007/orders?state=paied` + +### Pagination + +You may use a range query parameter. Pagination is mandatory : a default pagination has to be defined, for example: range=0-25. +The response should contain the following headers: `Link`, `Content-Range`, `Accept-Range`. +Note that pagination may cause some unexpected behavior if many resources are added. + +```sh +/orders?range=48-55 +206 Partial Content +Content-Range: 48-55/971 +Accept-Range: order 10 +Link : ; rel="first", +; rel="prev", +; rel="next", +; rel="last" +``` + +### Partial responses + +You should use partial responses so developers can select informaTon needed and optimize bandwidth (mobile development). + +```sh +GET /users/007?fields=firstname,name,address(street) + +200 OK + +{ + "id":"007", + "firstname":"James", + "name":"Bond", + "address": { + "street":"Horsen Ferry Road" + } +} +``` + +### Sort + +Use `?sort =atribute1,atributeN` to sort resources. By default resources are sorted in ascending order. +Use `?desc=atribute1,atributeN` to sort resources in descending order +`GET /restaurants?sort=rating,reviews,name;desc=rate,reviews` + +```sh +# URL reserved words : first, last, count +# Use `/first` to get the 1st element +GET /orders/first +200 OK +{"id":"1234", "state":"paid"} + + +# Use `/last` to retrieve the latest resource of a collection +GET /orders/last +200 OK +{"id":"5678", "state":"running"} ! + +# Use `/count` to get the current size of a collection +GET /orders/count !! +200 OK ! +{"2"} +``` + +--- + +## Other concepts + +### Content negotiation + +Content negotiation is managed only in a pure RESTful way. The client asks for the requierd content, in the Accept Header, in order of preference. Default format is JSON. Accept: application/json, text/plain not /orders.json! + +### I18N + +Use ISO 8601 standard for Date/Time/Timestamp: `1978-05-10T06:06:06+00:00` or `1978-05-10` +Add support for different Languages. Accept-Language: `fr-ca, fr-fr` not `?language=fr` + +### Cross-origin requests + +Use `CORS` standard to support REST API requests from browsers (js SPAโ€ฆ). +But if you plane to support Internet Explorer 7/8 or 9, you shall consider specifics endpoints to add a `jsonp` support. +- All requests will be sent with a GET method! +- Content negoTation cannot be handled with Accept Header in Jsonp. +- Payload cannot be used to send data. ! + +`POST /orders` and `/orders.jsonp?method=POST&callback=foo` +`GET /orders` and `/orders.jsonp?callback=foo` +`GET /orders/1234` and `/orders/1234.jsonp?callback=foo` +`PUT /orders/1234` and `/orders/1234.jsonp?method=PUT&callback=foo` + +Warning: a web crawler could easily damage your application with a method parameter. Make sure that an OAuth2 `access_token` is required, and an OAuth2 `client_id` as well. + +### HATEOAS + +Your API should propose Hypermedia links in order to be completely discoverable. But keep in mind that a majority of users wont probably use those hyperlinks, and will read the API documentation and copy/paste call examples. So, each call to the API should return in the Link Header every possible state of the application from the current state, plus self. You may use `RFC5988` Link notation to implement HATEOAS: + +```sh +GET /users/007 +< 200 Ok +< { "id":"007", "firstname":"James",...} +< Link : ; rel="self"; method:"GET", +; rel="addresses"; method:"GET", +; rel="orders"; method:"GET" +``` + +### "Non-Resources" scenarios + +In a few use cases we have to consider operations or services rather than resources. +You may use a `POST` request with a verb at the end of the URI +`POST /emails/42/send` +`POST /calculator/sum [1,2,3,5,8,13,21]` +`POST /convert?from=EUR&to=USD&amount=42` + +## see also + +- [[curl]] +- [[acid crud]] + diff --git a/notes/api docker registry.md b/notes/api docker registry.md new file mode 100644 index 00000000..c0b265cf --- /dev/null +++ b/notes/api docker registry.md @@ -0,0 +1,27 @@ +--- +tags: [container] +title: api docker registry +created: '2019-07-30T06:19:49.043Z' +modified: '2023-03-22T10:23:02.551Z' +--- + +# api docker registry + +## usage + +```sh +HEAD /v2//blobs/ + +GET /v2/path/manifests/0.0.2-SNAPSHOT + +GET /v2/path/manifests/latest + +curl --head -k -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -u user:pass "https://foo.bar.baz/v2/path/manifests/0.0.2-SNAPSHOT" +``` + +## see also + +- [[docker]] +- [[nexus api]] + diff --git a/notes/api docker.md b/notes/api docker.md new file mode 100644 index 00000000..5fe7f238 --- /dev/null +++ b/notes/api docker.md @@ -0,0 +1,64 @@ +--- +tags: [container] +title: api docker +created: '2019-08-20T09:42:39.909Z' +modified: '2023-03-22T10:23:02.539Z' +--- + +# api docker + +## usage + +```sh +-H "unix:///var/run/docker.sock" # docker-daemon listen unix socket + +curl --unix-socket /var/run/docker.sock \ + "http://./debug/pprof/goroutine?debug=2" # dump stacktrace + +curl --unix-socket /var/run/docker.sock \ + "http://containers/json" # start container + +curl -XPOST --unix-socket /var/run/docker.sock \ + -H 'Content-Type: application/json' \ + -d '{"Image":"nginx"}' \ + "http://localhost/containers/create" +# { "Id": "fcb65c6147efcb6...7d65", "Warnings":null } + +curl -XPOST --unix-socket /var/run/docker.sock \ + http://localhost/containers/fcb6...7d65/start + +# get events from inside container +docker run -ti \ # run in interactive-mode + -v /var/run/docker.sock:/var/run/docker.sock \ # (1) bind mounts the docker.sock + alpine sh +curl --unix-socket /var/run/docker.sock \ # (2) get events + "http://localhost/events" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock/Plugin.Activate \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/Plugin.Activate" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/Plugin.Activate" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/VolumeDriver.List" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/VolumeDriver.Path" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/VolumeDriver.Capabilities" + +curl -XGET --unix-socket /run/docker/plugins/nfs.sock \ + "http://%2Frun%2Fdocker%2Fplugins%2Fnfs.sock/VolumeDriver.Get" + +/Plugin.Activate +``` + +## see also + +- [[curl]] +- [[unix socket]] +- [[api kubernetes]] +- [docs.docker.com/engine/api](https://docs.docker.com/engine/api/v1.24/) +- [github.com/docker/go-plugins-helpers/](https://github.com/docker/go-plugins-helpers/) diff --git a/notes/api elasticsearch.md b/notes/api elasticsearch.md new file mode 100644 index 00000000..77f5dc2b --- /dev/null +++ b/notes/api elasticsearch.md @@ -0,0 +1,66 @@ +--- +tags: [elasticsearch] +title: api elasticsearch +created: '2019-09-03T09:09:54.970Z' +modified: '2023-03-22T10:23:02.560Z' +--- + +# api elasticsearch + +## usage + +```sh +GET /_cat/nodes?h=h,diskAvail + +GET /_cat/indices?v # show all indices open and close ! + +GET /_cat/indices/some-index_392?v + +GET /_cat/allocation?v + +GET /_cat/allocation?v&pretty&h=node,disk.indices,disk.used,disk.avail,disk.total,disk.percent + +GET /_cat/shards?v + +GET /_cat/shards?v&h=index,shard,docs,store,node + +?help +?bytes=b +# `b` Bytes +# `kb` Kilobytes +# `mb` Megabytes +# `gb` Gigabytes +# `tb` Terabytes +# `pb` Petabytes +?s=order:desc,index_patterns + + +GET /_nodes/process?pretty # e.g. find out mlockall: true + +GET /_nodes/_all/settings?flat_settings + + +GET /_cluster/health?pretty # status + +GET /_cluster/state/metadata + +GET /_cluster/state/blocks + + +GET /_cluster/settings + +PUT /_cluster/settings '{ "transient": { "cluster.routing.allocation.disk.watermark.low": "90%", "cluster.routing.allocation.disk.watermark.high": "90%" } }' +# disk-based shard allocation +# cluster.routing.allocation.disk.watermark.low # (85%) will not allocate shards to nodes that have more than 85% disk used. +# cluster.routing.allocation.disk.watermark.high # (90%) attempt to relocate shards away from a node whose disk usage is above 90%. +# cluster.routing.allocation.disk.watermark.flood_stage # (95%) enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node that has at least one disk exceeding the flood stage. This is a last resort to prevent nodes from running out of disk space. The index block must be released manually once there is enough disk space available to allow indexing operations to continue. + +PUT /_cluster/settings '{ "transient" :{ "cluster.routing.allocation.exclude._ip" : "10.20.30.40" } }' # remove node from cluster +``` + +## see also + +- [[curator]] +- [[ulimit]] +- [reference/current/cat.html](https://www.elastic.co/guide/en/elasticsearch/reference/current/cat.html) + diff --git a/notes/api github.md b/notes/api github.md new file mode 100644 index 00000000..078fc50f --- /dev/null +++ b/notes/api github.md @@ -0,0 +1,34 @@ +--- +tags: [cloud] +title: api github +created: '2020-11-20T23:42:32.328Z' +modified: '2023-05-24T08:45:01.604Z' +--- + +# api github + +> + +## usage + +```sh +api.github.com + +GET /users/ORG/orgs + +GET /orgs/ORG/repos + +GET /users/USER/repos?type=owner | jq -r '.[] | select(.fork == false)|.ssh_url' + + +curl --user "TOKEN:x-oauth-basic" "https://api.github.com/user" + +curl --user "TOKEN:x-oauth-basic" "https://api.github.com/orgs/ORG/repos" | jq -r '.[].ssh_url' +``` + +## see also + +- [developer.github.com/v3](https://developer.github.com/v3/) +- [[api gitlab]] +- [[jq]] +- [[gh]] diff --git a/notes/api gitlab.md b/notes/api gitlab.md new file mode 100644 index 00000000..23cbe4bf --- /dev/null +++ b/notes/api gitlab.md @@ -0,0 +1,29 @@ +--- +tags: [cloud] +title: api gitlab +created: '2023-03-11T10:08:17.544Z' +modified: '2023-05-24T08:44:54.814Z' +--- + +# api gitlab + +## usage + +```sh +curl "https://gitlab.example.com/api/v4/projects" + +curl --header "Authorization: Bearer OAUTH-TOKEN" "https://gitlab.example.com/api/v4/projects" +curl --header "PRIVATE-TOKEN: " "https://gitlab.example.com/api/v4/projects" +curl --header "Authorization: Bearer " "https://gitlab.example.com/api/v4/projects" + + +GET "/api/v4/namespaces?per_page=50" + +GET "/api/v4/projects?pagination=keyset&per_page=100&order_by=id&sort=asc" +``` + +## see also + +- [[api github]] +- [[curl]] +- [[glab]] diff --git a/notes/apk.md b/notes/apk.md index d9e429e8..526fea86 100644 --- a/notes/apk.md +++ b/notes/apk.md @@ -1,38 +1,35 @@ --- -tags: [container/docker, linux/packagemanager] +tags: [container, linux, packagemanager, shell] title: apk created: '2019-07-30T20:26:52.476Z' -modified: '2019-07-30T20:37:01.456Z' +modified: '2023-03-27T05:51:34.613Z' --- # apk -> Software packages for `Alpine Linux` are digitally signed `tar.gz` archives containing programs, configuration files, and dependency metadata. They have the extension `.apk`, and are often called `"a-packs"` +> packages for `Alpine Linux` are digitally signed `tar.gz` archives containing programs, configuration files, and dependency metadata. They have the extension `.apk`, and are often called `"a-packs"` -[wiki.alpinelinux.org](https://wiki.alpinelinux.org/wiki/Alpine_Linux_package_management) +## usage -## list installed ```sh -apk info -``` +apk info # list installed -## install package -```sh -apk add --no-cache package -``` +apk add --no-cache package # install package -### unsatisfiable constraints -if package is only available in edge alpine repo, not in a stable one -```sh -echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories -``` -[docker - ERROR: unsatisfiable constraints using apk in dockerfile - Stack Overflow](https://stackoverflow.com/a/48893148) +apk add --update py-pip +apk del pkg ---- +# unsatisfiable constraints +# if package is only available in edge alpine repo, not in a stable one +echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +``` +## see also -[docker-alpine :: viewdocs.io](http://gliderlabs.viewdocs.io/docker-alpine/) -[BusyBox - The Swiss Army Knife of Embedded Linux](https://busybox.net/downloads/BusyBox.html) +- [wiki.alpinelinux.org](https://wiki.alpinelinux.org/wiki/Alpine_Linux_package_management) +- [docker-alpine :: viewdocs.io](http://gliderlabs.viewdocs.io/docker-alpine/) +- [BusyBox - The Swiss Army Knife of Embedded Linux](https://busybox.net/downloads/BusyBox.html) +- [ERROR: unsatisfiable constraints using apk in dockerfile - Stack Overflow](https://stackoverflow.com/a/48893148) diff --git a/notes/apropos.md b/notes/apropos.md new file mode 100644 index 00000000..592bf07d --- /dev/null +++ b/notes/apropos.md @@ -0,0 +1,26 @@ +--- +tags: [linux, macos] +title: apropos +created: '2019-09-23T05:22:20.385Z' +modified: '2023-05-19T11:12:34.655Z' +--- + +# apropos + +> search the whatis database for strings + +## usage + +```sh +apropos namespace # searches for man pages with namespaces + +apropos -e fork # limit exact word +apropos "^fork$" +``` + +## see also + +- [[man]], [[whatis]] +- [[which]] +- [[type]] +- [[command-not-found]] diff --git a/notes/apt-cache.md b/notes/apt-cache.md new file mode 100644 index 00000000..8214afae --- /dev/null +++ b/notes/apt-cache.md @@ -0,0 +1,32 @@ +--- +tags: [linux] +title: apt-cache +created: '2019-11-28T11:57:30.886Z' +modified: '2020-04-21T08:30:28.253Z' +--- + +# apt-cache + +> performs a variety of operations on apt's package cache - does not manipulate the state of the system, but provides operations to search and generate interesting output from the package metadata + +## usage + +```sh +apt-cache search REGEX # full text search on available package lists; searches package names and descriptions + # --full output identical to show is produced for each matched package + # --names-only long description is not searched, only the package name + +apt-cache search lint + +apt-cache search $i | grep -E "^$i\s" + +apt-cache madison rabbitmq-server + +apt-cache showpkg + +apt-cache policy +``` + +## see also + +- [[apt]] diff --git a/notes/apt-file.md b/notes/apt-file.md new file mode 100644 index 00000000..f39bc116 --- /dev/null +++ b/notes/apt-file.md @@ -0,0 +1,27 @@ +--- +tags: [linux] +title: apt-file +created: '2020-01-16T07:31:31.148Z' +modified: '2022-04-06T11:35:45.799Z' +--- + +# apt-file + +> search for file/binary in packages + +## install + +`apt install apt-file` + +## usage + +```sh +apt-file update + +apt-file search mkpasswd +``` + +## see also + +- [[apt]] +- [[apt-get]] diff --git a/notes/apt-get.md b/notes/apt-get.md new file mode 100644 index 00000000..2b58eafe --- /dev/null +++ b/notes/apt-get.md @@ -0,0 +1,28 @@ +--- +tags: [linux, packagemanager] +title: apt-get +created: '2021-10-29T12:42:44.200Z' +modified: '2022-02-02T09:04:53.032Z' +--- + +# apt-get + +> apt package handling utility + +## usage + +```sh +apt-get update + +apt-get upgrade + +apt-get install PACKAGE +``` + +## see also + +- [[apt]] +- [[apt-key]] +- [[brew]] +- [[yum]] +- [[zypper]] diff --git a/notes/apt-key.md b/notes/apt-key.md new file mode 100644 index 00000000..ca3a4f9d --- /dev/null +++ b/notes/apt-key.md @@ -0,0 +1,47 @@ +--- +tags: [linux] +title: apt-key +created: '2022-02-02T08:54:23.749Z' +modified: '2023-07-19T11:17:06.469Z' +--- + +# apt-key + +> manage list of keys used by apt to authenticate packages + +`apt-key` supports only the binary `openPGP` format (`"GPG key public ring"`) in files with the [[gpg]] extension + +## option + +```sh +-v, --version # get version +-h, --help # get help + --keyring FILE # +``` + +## usage + +```sh +apt-key add FILE # add a new key to the list of trusted keys + +apt-key del KEY_ID # Remove a key from the list of trusted keys + +apt-key export KEY_ID # output the key keyid to standard output + +apt-key exportall # output all trusted keys to standard output + +apt-key list, finger # list trusted keys with fingerprints + +apt-key adv # pass advanced options to gpg, with adv + +apt-key adv \ + --keyserver keyserver.ubuntu.com \ + --recv-keys KEY_ID # --recv-key: download key from keyservers directly into the trusted set of keys +``` + +## see also + +- [[gpg]] +- [[apt]] +- [[apt-get]] +- [[add-apt-repository]] diff --git a/notes/apt.md b/notes/apt.md index e1c5505a..ff03c87b 100644 --- a/notes/apt.md +++ b/notes/apt.md @@ -1,55 +1,33 @@ --- -tags: [linux/packagemanager] +tags: [linux, packagemanager, python] title: apt created: '2019-07-30T20:20:43.614Z' -modified: '2019-07-30T20:35:53.347Z' +modified: '2023-06-28T08:24:27.222Z' --- # apt -### used packages -```sh -apt install net-tools # ifconfig, nslookup -apt install dnsutils # dig -apt install iputils-ping # ping -apt install iproute2 # ip -``` +> high-level commandline interface for the package management system, frontend to [[apt-get]] -### repository commands -```sh -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DF7DD7A50B746DD4 +## usage -add-apt-repository "deb https://download.srcclr.com/ubuntu stable/" -add-apt-repository ppa:ian-berke/ppa-drawers # ppa +```sh +apt update -apt-get update -apt-get install pkg +apt install PKG -add-apt-repository -r "deb https://download.srcclr.com/ubuntu stable/" +apt remove PKG -apt remove drawers apt autoremove # remove former dependencies -add-apt-repository -r ppa:ian-berke/ppa-drawers - -# apt-cache -apt-cache madison rabbitmq-server -apt-cache showpkg - -apt-cache policy +apt list --installed ``` -### search for packages ?! -```sh -#!/bin/bash - -# declare -a arrayName=(...) -package=('cmake' 'valac' 'libgtk-3-dev' 'librest-dev' 'libjson-glib-dev' 'libnotify-dev' 'libcanberra-dev' 'libx11-dev' 'libwebkitgtk-3.0-dev' 'libsqlite3-dev' 'libxtst-dev' 'libpurple-dev' 'libgee-dev' 'libdbusmenu-gtk-dev' 'libgtksourceview-3.0-dev') +## see also -echo 'array Size:' ${#package[@]} - -for i in "${package[@]}" do - : - echo $(apt-cache search $i | grep -E "^$i\s") -done -``` +- [[apt-get]] +- [[apt-key]] +- [[apt-file]] +- [[apt-cache]] +- [[brew]] +- [[nix]] diff --git a/notes/architectural pattern.md b/notes/architectural pattern.md new file mode 100644 index 00000000..a2896b2e --- /dev/null +++ b/notes/architectural pattern.md @@ -0,0 +1,15 @@ +--- +tags: [Notebooks] +title: architectural pattern +created: '2020-03-12T09:08:11.760Z' +modified: '2022-04-06T11:36:03.247Z' +--- + +# architectural pattern + +> Even though an architectural pattern conveys an image of a system, it is not an architecture. An architectural pattern is a concept that solves and delineates some essential cohesive elements of a software architecture. Countless different architectures may implement the same pattern and share the related characteristics. Patterns are often defined as "strictly described and commonly available" + +## see also + +- [[hexagonal architecture]] +- [[software design pattern]] diff --git a/notes/argocd.md b/notes/argocd.md new file mode 100644 index 00000000..181ae7fe --- /dev/null +++ b/notes/argocd.md @@ -0,0 +1,40 @@ +--- +tags: [container] +title: argocd +created: '2020-10-26T12:29:59.372Z' +modified: '2022-03-16T07:10:31.142Z' +--- + +# argocd + +> declarative, gitops continuous delivery tool for k8s + +## install + +```sh +brew install argocd # install cli + +kubectl create namespace argocd # install argocd on cluster +kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/core-install.yaml +``` + +## usage + +```sh +argocd login HOST + +argocd app create guestbook \ + --repo https://github.com/argoproj/argocd-example-apps.git \ + --path guestbook \ + --dest-server https://kubernetes.default.svc \ + --dest-namespace default + +argocd app list +``` + +## see also + +- [[flux]] +- [[gitops]] +- [[kubectl]] +- [[minikube]] diff --git a/notes/arp.md b/notes/arp.md new file mode 100644 index 00000000..33249a3a --- /dev/null +++ b/notes/arp.md @@ -0,0 +1,45 @@ +--- +tags: [linux, net-tools, network] +title: arp +created: '2019-09-03T11:30:41.018Z' +modified: '2023-03-22T08:28:25.338Z' +--- + +# arp + +> manipulates or displays the kernel's ipv4 network neighbour cache. It can add entries to the table, delete one, or display the current content. + +## install + +```sh +apt install net-tools +``` + +## usage + +```sh +arp -a # display (all) hosts in alternative (BSD) style + +arp -a | awk '{ ahost=$1; aip=$2; gsub("()","",aip); print aip }' + +arp -a | awk '{ gsub(/[()]/,""); print $1 " " $2}' + + +arp -e # display (all) hosts in default (Linux) style + +arp -a 159.254.171.22 + +arp -i ens192 # display for interface + + +cat /proc/net/arp +``` + +## see also + +- [[arping]] +- [[procfs]] +- [[ip]] +- [[net-tools vs iproute]] +- [omputerhope.com/unix/arp](https://www.computerhope.com/unix/arp.htm) +- [superuser.com/questions/1272010/where-is-the-arp-cache-on-linux](https://superuser.com/questions/1272010/where-is-the-arp-cache-on-linux) diff --git a/notes/arping.md b/notes/arping.md new file mode 100644 index 00000000..1c39e84c --- /dev/null +++ b/notes/arping.md @@ -0,0 +1,32 @@ +--- +tags: [linux] +title: arping +created: '2019-12-26T19:30:34.765Z' +modified: '2023-03-25T12:45:27.436Z' +--- + +# arping + +> send arp requests to a neighbour host + +## install + +```sh +apt install arping +yum install arping +``` + +## usage + +```sh +arping # send arp request to a neighbour host + +arping -I eth0 192.168.1.1 # send arp request to 192.168.1.1 via interface eth0 + +arping -D -I eth0 192.168.1.1 # check for duplicate MAC addresses at 192.168.1.1 on eth0 +``` + +## see also + +- [[arp]] +- [[ping]] diff --git a/notes/as.md b/notes/as.md new file mode 100644 index 00000000..95a9b893 --- /dev/null +++ b/notes/as.md @@ -0,0 +1,27 @@ +--- +tags: [macos] +title: as +created: '2023-05-09T06:57:12.701Z' +modified: '2023-05-09T06:58:58.845Z' +--- + +# as + +> macos X Mach-O GNU-based assemblers - translates assembly code in the named files to object code + +## option + +```sh +-o name # name the output file name instead of a.out +``` + +## usage + +```sh +as -o FILE.o FILE.as +``` + +## see also + +- [[ld]] +- [[make]] diff --git a/notes/ascii.md b/notes/ascii.md index 4c79cf41..d9949b08 100644 --- a/notes/ascii.md +++ b/notes/ascii.md @@ -1,91 +1,85 @@ --- -tags: [encoding/ascii] +tags: [linux, macos, Notebooks] title: ascii created: '2019-07-30T06:19:48.987Z' -modified: '2019-07-30T09:06:14.044Z' +modified: '2023-05-19T11:04:19.568Z' --- # ascii -> `ASCII - American Standard Code for Infromation Interchange` is a character `encoding`standard for electronic communication +> `american standard code for infromation interchange` - character `encoding` standard +`codepoint` - numeric value that maps to a character +`codespace` - set of all codepoints -## ascii character set +ascii is comprised of 128 `codepoints` from `0x00` to `0x7F` -```sh -man ascii -``` +## install -### print all avail. characters ```sh -for ((i=32;i<127;i++)) do - printf "\\$(printf %03o "$i")"; -done -printf "\n" +brew install ascii ``` - -| char | oct | hex |dec | -| :-- | :-- | :-- |:-- | -| `"` | 042 | 22 | 34 | -| `J` | 112 | 4a | 74 | +## option ```sh -printf '\112' # or -echo $'\112' # octal - -printf '\x4a' # hex - -printf "\\$(printf %o 74)" # or -xxd -r <<<'0 4a' # decimal +-t # one-line output +-a # vertical format +-d # Decimal table +-o # octal table +-x # hex table +-b # binary table +-h # This help screen +-v # version information ``` +## usage -## get decimal-set ```sh -echo '"' | tr -d "\n" | od -An -t uC -# | โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Use od (octal dump) to print: -# | -An means Address none -# remove "newline" char -t select a type -# u type is unsigned decimal. -# C of size (one) char - +ascii -x # get table in hex +man 7 ascii # table hexadecimal ``` -## hexdump file -```sh -xxd docker-compose.yml # hexdump of file ``` -[Jafrog's dev blog](http://jafrog.com/2013/11/23/colors-in-terminal.html) - - -## table hexadecimal - +char | oct | hex | dec +" | 042 | 22 | 34 +J | 112 | 4a | 74 ``` -# _0 _1 _2 _3 _4 _5 _6 _7 _8 _9 _A _B _C _D _E _F - -# 0_ NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI -# 0 0000 0001 0002 0003 0004 0005 0006 0007 0008 0009 000A 000B 000C 000D 000E 000F -# 1_ DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US -# 16 0010 0011 0012 0013 0014 0015 0016 0017 0018 0019 001A 001B 001C 001D 001E 001F +## hex -# 2_ SP ! " # $ % & ' ( ) * + , - . / -# 32 0020 0021 0022 0023 0024 0025 0026 0027 0028 0029 002A 002B 002C 002D 002E 002F - -# 3_ 0 1 2 3 4 5 6 7 8 9 : ; < = > ? -# 48 0030 0031 0032 0033 0034 0035 0036 0037 0038 0039 003A 003B 003C 003D 003E 003F - -# 4_ @ A B C D E F G H I J K L M N O -# 64 0040 0041 0042 0043 0044 0045 0046 0047 0048 0049 004A 004B 004C 004D 004E 004F - -# 5_ P Q R S T U V W X Y Z [ \ ] ^ _ -# 80 0050 0051 0052 0053 0054 0055 0056 0057 0058 0059 005A 005B 005C 005D 005E 005F - -# 6_ ` a b c d e f g h i j k l m n o -# 96 0060 0061 0062 0063 0064 0065 0066 0067 0068 0069 006A 006B 006C 006D 006E 006F - -# 7_ p q r s t u v w x y z { | } ~ DEL -# 112 0070 0071 0072 0073 0074 0075 0076 0077 0078 0079 007A 007B 007C 007D 007E 007F ``` +00 nul 01 soh 02 stx 03 etx 04 eot 05 enq 06 ack 07 bel +08 bs 09 ht 0a nl 0b vt 0c np 0d cr 0e so 0f si +10 dle 11 dc1 12 dc2 13 dc3 14 dc4 15 nak 16 syn 17 etb +18 can 19 em 1a sub 1b esc 1c fs 1d gs 1e rs 1f us +20 sp 21 ! 22 " 23 # 24 $ 25 % 26 & 27 ' +28 ( 29 ) 2a * 2b + 2c , 2d - 2e . 2f / +30 0 31 1 32 2 33 3 34 4 35 5 36 6 37 7 +38 8 39 9 3a : 3b ; 3c < 3d = 3e > 3f ? +40 @ 41 A 42 B 43 C 44 D 45 E 46 F 47 G +48 H 49 I 4a J 4b K 4c L 4d M 4e N 4f O +50 P 51 Q 52 R 53 S 54 T 55 U 56 V 57 W +58 X 59 Y 5a Z 5b [ 5c \ย  5d ] 5e ^ 5f _ +60 ` 61 a 62 b 63 c 64 d 65 e 66 f 67 g +68 h 69 i 6a j 6b k 6c l 6d m 6e n 6f o +70 p 71 q 72 r 73 s 74 t 75 u 76 v 77 w +78 x 79 y 7a z 7b { 7c | 7d } 7e ~ 7f del +``` + +## see also + +- [[bash printf]] +- [[bash echo]] +- [[man]] +- [[od]] +- [[xxd]] +- [[iconv]] +- [[baudot]] +- [[markdown]] +- [[url encoding]] +- [[hyper text transfer protocol]] +- [asciimoji.com](http://asciimoji.com/) +- [asciiflow.com](https://asciiflow.com/#/) +- [jafrog.com/2013/11/23/colors-in-terminal](http://jafrog.com/2013/11/23/colors-in-terminal.html) diff --git a/notes/asdf.md b/notes/asdf.md new file mode 100644 index 00000000..0c5aa2ab --- /dev/null +++ b/notes/asdf.md @@ -0,0 +1,37 @@ +--- +tags: [shell] +title: asdf +created: '2021-03-29T06:54:55.237Z' +modified: '2023-03-23T07:42:12.313Z' +--- + +# asdf + +> manage multiple language runtime versions on a per-project basis - like [[gvm]], [[nvm]], [[rbenv]],[[pyenv]], .. all in one! Simply install your language's plugin! + +## install + +```sh +brew install asdf +``` + +## usage + +```sh +asdf + +asdf plugin add nodejs https://github.com/asdf-vm/asdf-nodejs.git + +asdf list all nodejs + +asdf list all nodejs 14 + +asdf install nodejs latest +``` + + +## see also + +- [github.com/asdf-vm/asdf](https://github.com/asdf-vm/asdf) +- [[sdk]] +- [[nvm]] diff --git a/notes/ash.md b/notes/ash.md new file mode 100644 index 00000000..67008e6c --- /dev/null +++ b/notes/ash.md @@ -0,0 +1,22 @@ +--- +tags: [shell] +title: ash +created: '2021-05-12T08:58:17.934Z' +modified: '2023-03-22T09:24:17.335Z' +--- + +# ash + +> almquist shell + +## usage + +```sh +zsh +``` + +## see also + +- [[busybox]] +- [[bash]], [[dash]] +- [rosettacode.org/wiki/Almquist_Shell](https://rosettacode.org/wiki/Almquist_Shell) diff --git a/notes/asm assmbler.md b/notes/asm assmbler.md deleted file mode 100644 index 28554962..00000000 --- a/notes/asm assmbler.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -tags: [lang/asm] -title: asm assmbler -created: '2019-07-30T06:19:48.988Z' -modified: '2019-07-30T09:06:01.981Z' ---- - -# asm assmbler - - -``` -EAX,EBX,ECX,EDX - "general purpose", more or less interchangeable - -lea rbx, qword [..] - -``` - -```c -unsigned long a1, r; -void junk( void ) -{ - asm( - "pushl %eax \n" - "pushl %ebx \n" - "movl $100,%eax \n" - "movl a1,%ebx \n" - "int $69 \n" - "movl %eax,r \n" - "popl %ebx \n" - "popl %eax \n" - ); -} - -``` - - diff --git a/notes/asm.md b/notes/asm.md new file mode 100644 index 00000000..a4ad9799 --- /dev/null +++ b/notes/asm.md @@ -0,0 +1,67 @@ +--- +tags: [c] +title: asm +created: '2019-07-30T06:19:48.988Z' +modified: '2022-04-06T11:36:41.806Z' +--- + +# asm + +> assembly language - closely tied to processor architecture + +- `CISC` Complex Instruction-Set Computer +- `RISC` Reduced Instruction-Set Computer +- `DSP` Digital Signal Processor +- `VLIW` Very Long Instruction Word + +## usage + +```asm +; opcode operands +add R1, R2, 3 + +; opcodes: arithmetic/logical +add +sub +mult +Cmp + +; opcodes: memory load/store +ld +st + +; opcodes: control transfer +jmp +bne + +; opcodes: complex +movs + + +EAX,EBX,ECX,EDX - "general purpose", more or less interchangeable + +lea rbx, qword [..] + +``` + +```c +unsigned long a1, r; + +void junk( void ) { + asm( "pushl %eax \n" + "pushl %ebx \n" + "movl $100,%eax \n" + "movl a1,%ebx \n" + "int $69 \n" + "movl %eax,r \n" + "popl %ebx \n" + "popl %eax \n" ); +} +``` + +## see also + +- [[wasm]] +- [[c]] +- [[rust]] + diff --git a/notes/atlantis.md b/notes/atlantis.md new file mode 100644 index 00000000..aff5e889 --- /dev/null +++ b/notes/atlantis.md @@ -0,0 +1,26 @@ +--- +tags: [iac] +title: atlantis +created: '2022-04-19T08:16:22.864Z' +modified: '2023-03-22T10:33:37.859Z' +--- + +# atlantis + +> terraform pull request automation + +## usage + +```sh +atlantis server \ + --atlantis-url="$URL" \ + --gh-user="$USERNAME" \ + --gh-token="$TOKEN" \ + --gh-webhook-secret="$SECRET" \ + --repo-allowlist="$REPO_ALLOWLIST" +``` + +## see also + +- [[terraform]] +- [github.com/runatlantis/atlantis](https://github.com/runatlantis/atlantis) diff --git a/notes/atom.md b/notes/atom.md index 321c0172..c0dabeb1 100644 --- a/notes/atom.md +++ b/notes/atom.md @@ -2,30 +2,38 @@ tags: [editor] title: atom created: '2019-07-30T06:19:48.989Z' -modified: '2019-07-30T20:37:25.171Z' +modified: '2022-03-04T07:42:47.142Z' --- # atom -cmd + shift + p # command palette will pop up +> editor based on electron -cmd + ; # open setting view +## usage -cmd + o # open project - - -## atom package manager ```sh -apm install project-manager - -apm install language-terraform - -apm install remember-session +cmd + shift + p # command palette will pop up +cmd + ; # open setting view +cmd + o # open project +ctl + shift + p `Window:Reload` # command palette ``` -## test +## atom package manager +```sh +apm list # list installed packages + +apm install \ + project-manager remember-session atom-beautify \ + docker \ + linter \ + language-docker \ + language-terraform terraform-fmt linter-terraform-syntax \ +``` -ctl + shift + p -Window:Reload +## see also +- [[vim]] +- [[code]] +- [[idea]] +- [[npm]] diff --git a/notes/autoconf.md b/notes/autoconf.md new file mode 100644 index 00000000..1eec58ef --- /dev/null +++ b/notes/autoconf.md @@ -0,0 +1,43 @@ +--- +tags: [buildsystem, c] +title: autoconf +created: '2022-02-02T13:35:55.731Z' +modified: '2023-03-22T10:40:07.333Z' +--- + +# autoconf + +> generate configuration scripts + +## option + +```sh +# operation modes +-h, --help # print this help, then exit +-V, --version # print version number, then exit +-v, --verbose # verbosely report processing +-d, --debug # don't remove temporary files +-f, --force # consider all files obsolete +-o, --output=FILE # save output in FILE (stdout is the default) +-W, --warnings=CATEGORY # report the warnings falling in CATEGORY + +# library directories +-B, --prepend-include=DIR # prepend directory DIR to search path +-I, --include=DIR # append directory DIR to search path + +# tracing +-t, --trace=MACRO[:FORMAT] # report the list of calls to MACRO +-i, --initialization # also trace autoconf's initialization process +``` + +## usage + +```sh +autoreconf -fi # generate config script +``` + +## see also + +- [[make]] +- [[automake]] +- [[curl]] diff --git a/notes/automake.md b/notes/automake.md new file mode 100644 index 00000000..80f28603 --- /dev/null +++ b/notes/automake.md @@ -0,0 +1,27 @@ +--- +tags: [buildsystem, c] +title: automake +created: '2019-08-21T14:37:43.236Z' +modified: '2023-05-01T13:14:21.485Z' +--- + +# automake + +> generates `Makefile.in` for configure from `Makefile.am` + +> `Makefile.am` is a programmer-defined file and is used by `automake` to generate the `Makefile.in` file + +- `.am` stands for `automake` +- `.in` input for `configure` think of template + +## usage + +```sh +automake +``` + +## see also + +- [[make]] +- [[autoconf]] +- [Using Autotools](https://developer.gnome.org/anjuta-build-tutorial/stable/create-autotools.html.en) diff --git a/notes/awk snippets.md b/notes/awk snippets.md deleted file mode 100644 index 6d14f4f2..00000000 --- a/notes/awk snippets.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -tags: [lang/awk] -title: awk snippets -created: '2019-08-02T07:17:02.123Z' -modified: '2019-08-02T07:17:50.148Z' ---- - -# awk snippets - -```sh -awk '{if (NR!=1) {print} }' # skip first line - -awk '{print $NF}' # prints last field - -awk '{$1=$2=""; print $0}' # don't print fields $1 and $2 - -awk '$3 ~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/ {print $1,$3}' # print where $3 has an IPv4 - -tar tvf scripts.tar | awk -F/ '{if (NF<4) print }' # print only first level of files - -awk '(NR%2 && /pattern/) || (!(NR%2) && /anotherpattern/)' # (NR%2) even and !(NR%2) uneven - -awk 'BEGIN{ FS=";" }{ print $6 }' - -awk -F'";"' '{ if (length($6)>31) print length($6),$6 }' - -awk '{ if( $1 ~ "swarm" && $3 == "alive") { split($2, arr, ":"); print arr[1]} }' # split '10.32.23.150:8301' -``` - -### csv -```sh -awk 'BEGIN {FS=","} ($15 != "") && ($19 == "10mbps SDSL" || $20 == "40mbps SDSL") { print .. }' - -awk 'BEGIN{FS=";"; TOTAL=0.00}{ if(NR!=1) { - gsub(/\./,""); # remove dot from numbers like 4.000,00 - gsub(/,/,"."); # replace , with . due to locale problems -.- - TOTAL +=sprintf("%f",$4); - printf "%s %.2f %.2f\n",$4,$4,TOTAL - } - }END{ printf "%.2f\n",TOTAL }' - -awk '{ split($1,swm,"."); split($2,svs,"_"); tmp = svs[1]" "services[swm[1]]; services[swm[1]]=tmp; }END{ - for (i in services) { - n=split(services[i], foo, " "); - print i, n, services[i] - } - }' -``` diff --git a/notes/awk.md b/notes/awk.md index 697b4e51..a7b80c92 100644 --- a/notes/awk.md +++ b/notes/awk.md @@ -1,67 +1,149 @@ --- -tags: [lang/awk, linux] +tags: [linux, macos] title: awk created: '2019-07-30T06:19:48.989Z' -modified: '2019-08-02T07:17:00.040Z' +modified: '2023-03-23T10:13:22.289Z' --- # awk -* initially developed in 1977 by `A`lfred Aho, Peter `W`einberger, and Brian `K`ernighan -* [Learn awk in Y Minutes](https://learnxinyminutes.com/docs/awk/) + +> domain-specific-language for text processing, data extraction and reporting +> data-driven scripting language +> name derived from authors surnames: Alfred Aho, Peter Weinberger, and Brian Kernighan ### implementations -``` -nawk # โ€œnew awkโ€, an evolution of oawk, the original UNIX implementation, used on *BSD and widely available on Linux -mawk # a fast implementation that mostly sticks to standard features -gawk # the GNU implementation, with many extensions -Busybox # small, intended for embedded systems, not many features -```` + +- [[nawk]]: `new awk` (evolution of `oawk`), the original unix implementation +- [[mawk]]: a fast implementation that mostly sticks to standard features +- [[gawk]]: the gnu implementation, with many extensions +- [[busybox]]: small, intended for embedded systems, not many features + +## option + ```sh -awk -f # Read the AWK program source from file +-f SCRIPT # read awk program from file -awk -F # Field Seperator +-F ";" # field seperator ``` -### Built-in Variables +## variables + +```sh +# 2 types +# variable which defines values which can be changed such as field separator `FS` and record separator `RS` +# variable which can be used for processing and reports such as Number of records, number of fields + +CONVFMT # conversion format used when converting numbers (default %.6g) +FS # input field separator; regular expression used to separate fields; also settable by option -Ffs. +NF # number of fields in the current record +NR # ordinal number of the current record +FNR # ordinal number of the current record in the current file + # Number of Records relative to the current input file / hen using two input files => seperate RecordNumbers + +FILENAME # the name of the current input file +RS # input record separator (default newline) +OFS # output field separator (default blank) +ORS # output record separator (default newline) +OFMT # output format for numbers (default %.6g) +SUBSEP # separates multiple subscripts (default 034) +ARGC # argument count, assignable +ARGV # argument array, assignable; non-null members are taken as filenames +ENVIRON # array of environment variables; subscripts are names. ``` -# 1. Type: Variable which defines values which can be changed such as field separator and record separator. -# 2. Type: Variable which can be used for processing and reports such as Number of records, number of fields. +## functions -FS # Input field separator variable +```sh +printf "%10.0f\n" 1.28071e+09 # print scientific notation as float to stdout -OFS # Output Field Separator Variable +var = fprintf("%s %d %.2f\n", "Testing", 1, 3) } # assigns its output to a variable, not stdout -RS # Record Separator variable +split($1,swm,"."); # split $1 into array swm[] with optional seperator "." -ORS # Output Record Separator Variable +substr("foobar", 2, 3) # => "oob" -NR # Number of Records Variable / aka line +substr("foobar", 4) # => "bar" -NF # Number of Fields in a record / aka $1 $2 .. +length("foo") # => 3 -FILENAME # Name of the current input file +tolower("FOO") # => "foo" -FNR # Number of Records relative to the current input file / when using two input files => seperate RecordNumbers -``` +toupper("foo") # => "FOO" -### Built-in Functions +gsub(/"/, "") #"# remove quotes globally ``` -printf "%10.0f\n" 1.28071e+09 # print scientific notation as float to stdout -var = fprintf("%s %d %.2f\n", "Testing", 1, 3) }' # assigns its output to a variable, not stdout +## usage + +```sh +awk '{$1=$1;print}' # trim whitespace from STDOUT + # works because assigning something to one of the fields, awk rebuilds the whole record + # by joining all fields ($1, ..., $NF) with OFS (space by default) +awk '{$1=$1};1' # shorter +awk '{$1=$1};NF' # also remove blank lines, NF: only print records for which the Number of Fields is non-zero + +awk '{if (NR!=1) {print} }' # skip first line + +awk '{print $NF}' # prints last field + +awk '{$1=$2=""; print $0}' # don't print fields $1 and $2 -gsub(/"/, "") # remove quotes globally +awk '$3 ~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/ {print $1,$3}' # print where $3 has an IPv4 -split($1,swm,"."); # split $1 into array swm[] with optional seperator "." +tar tvf scripts.tar | awk -F/ '{if (NF<4) print }' # print only first level of files -substr("foobar", 2, 3) # => "oob" +awk '(NR%2 && /pattern/) || (!(NR%2) && /anotherpattern/)' # (NR%2) even and !(NR%2) uneven -substr("foobar", 4) # => "bar" +awk 'BEGIN{ FS=";" }{ print $6 }' -length("foo") # => 3 +awk -F'";"' '{ if (length($6)>31) print length($6),$6 }' -tolower("FOO") # => "foo" +awk '{ if( $1 ~ "swarm" && $3 == "alive") { split($2, arr, ":"); print arr[1]} }' # split '10.32.23.150:8301' -toupper("foo") # => "FOO" + +echo foo.bar.bbaz.com | awk 'BEGIN{FS="."}{print substr($3,2),$1}' + + +# usage of arrays +awk '{array[$5] += $4}END{ for(var in array){ printf("%s: %s GB\n",var, array[var]/1024/1024/1024) } }' + +awk '{ node[1]+=$3; node[2]+=$4; node[3]+=$5; node[4]+=$6; }END{ + for(i in node){ printf("elastic-monitor-1-data-%s %s GB\n",i, node[i]/1024) } }' + + +# matching ipv4 pattern +awk 'BEGIN{ ORS="\n" }{ if ($3 ~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) { print $1} }' + +awk 'BEGIN{ NF=NF RS= OFS=+ }{ \ + if ($3 ~ /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/) { \ + gsub(/^[[:space:]]+|[[:space:]]+$/,"",$1); print $1} \ + }' + + +# reading csv +awk 'BEGIN {FS=","} ($15 != "") && ($19 == "10mbps SDSL" || $20 == "40mbps SDSL") { print .. }' + +awk 'BEGIN{FS=";"; TOTAL=0.00}{ if(NR != 1) { + gsub(/\./,""); # remove dot from numbers like 4.000,00 + gsub(/,/,"."); # replace , with . due to locale problems -.- + TOTAL +=sprintf("%f",$4); + printf "%s %.2f %.2f\n",$4,$4,TOTAL + } + }END{ printf "%.2f\n",TOTAL }' + +awk '{ + split($1,swm,"."); split($2,svs,"_"); tmp = svs[1]" "services[swm[1]]; services[swm[1]]=tmp; }END{ + for (i in services) { n=split(services[i], foo, " "); print i, n, services[i] } + }' + +# cursor bounce around the terminal +# Make your cursor bounce around the terminal. The 400000 in the for loop is just a busy delay. Adjust as needed. +yes $COLUMNS $LINES | awk 'BEGIN{x=y=e=f=1}{if(x==$1||!x){e*=-1};if(y==$2||!y){f*=-1};x+=e;y+=f;printf "\033[%s;%sH",y,x;for (a=0;a<400000;a++){}}' ``` + +## see also + +- [learnxinyminutes.com/docs/awk](https://learnxinyminutes.com/docs/awk/) +- [[sed]] +- [[grep]] +- [[column]] diff --git a/notes/aws-iam-authenticator.md b/notes/aws-iam-authenticator.md new file mode 100644 index 00000000..3727ae0e --- /dev/null +++ b/notes/aws-iam-authenticator.md @@ -0,0 +1,27 @@ +--- +tags: [cloud] +title: aws-iam-authenticator +created: '2022-03-24T07:18:31.601Z' +modified: '2023-03-24T08:24:48.571Z' +--- + +# aws-iam-authenticator + +> tool to use AWS IAM credentials to authenticate to Kubernetes cluster + +## install + +```sh +brew install aws-iam-authenticator +``` + +## usage + +```sh +aws-iam-authenticator help +``` + +## see also + +- [[aws]] +- [github.com/kubernetes-sigs/aws-iam-authenticator](https://github.com/kubernetes-sigs/aws-iam-authenticator) diff --git a/notes/aws-nuke.md b/notes/aws-nuke.md new file mode 100644 index 00000000..418d5cc9 --- /dev/null +++ b/notes/aws-nuke.md @@ -0,0 +1,73 @@ +--- +tags: [cloud] +title: aws-nuke +created: '2022-02-03T08:04:22.692Z' +modified: '2023-03-24T08:24:48.590Z' +--- + +# aws-nuke + +> remove all resources from an aws account + +## install + +```sh +wget -c https://github.com/rebuy-de/aws-nuke/releases/download/v2.16.0/aws-nuke-v2.16.0-linux-amd64.tar.gz -O - | sudo tar -xz -C $HOME/bin + +docker run --rm -it \ + -v $(pwd)/.aws:/home/aws-nuke/.aws \ + quay.io/rebuy/aws-nuke:v2.16.0 \ + --profile default \ + --config /home/aws-nuke/.aws/nuke-config.yml +``` + +## config + +```sh +cat < nuke-config.yaml +regions: +- eu-west-1 +- global + +account-blocklist: +- "999999999999" # production + +accounts: + "000000000000": {} # aws-nuke-example + "111100001111": + filters: + IAMUser: + - "admin" + IAMUserPolicyAttachment: + - "admin -> AdministratorAccess" + IAMUserAccessKey: + - "admin -> AKSDAFRETERSDF" + - "admin -> AFGDSGRTEWSFEY" + +resource-types: + excludes: + - IAMUser # don't nuke IAM users +EOF +``` + +## option + +```sh +--no-dry-run # to actually nuke resources +--access-key-id # access key +--secret-access-key # secret access key +``` + +## usage + +```sh +aws-nuke resource-types # list resources + +aws-nuke -c config/nuke-config.yml --profile aws-nuke-example +``` + +## see also + +- [[aws]] +- [[localstack]] + diff --git a/notes/aws.md b/notes/aws.md new file mode 100644 index 00000000..582cf080 --- /dev/null +++ b/notes/aws.md @@ -0,0 +1,551 @@ +--- +tags: [iac] +title: aws +created: '2019-07-30T06:19:48.990Z' +modified: '2022-12-05T14:51:00.473Z' +--- + +# aws + +> aws cli - unified tool to manage your aws services + +## install + +```sh +curl -LO "https://awscli.amazonaws.com/AWSCLIV2.pkg" && installer -pkg ./AWSCLIV2.pkg -target / +``` + +## environment + +```sh +AWS_ACCESS_KEY_ID # access key associated with an iam user or role +AWS_CA_BUNDLE # path to certificate bundle for HTTPS certificate validation +AWS_CLI_AUTO_PROMPT # enables auto-prompt, two settings can be used: on, on-partial +AWS_CLI_FILE_ENCODING # encoding used for text files +AWS_CONFIG_FILE # location of the file that the AWS CLI uses to store configuration profiles. default: ~/.aws/config +AWS_DATA_PATH # list of additional directories to check outside of the built-in search path of ~/.aws/models +AWS_DEFAULT_OUTPUT # output format: json, yaml, yaml-stream, text, table +AWS_DEFAULT_REGION # aws region to send the request to +AWS_EC2_METADATA_DISABLED # disables use of ec2 instance metadata service (IMDS) +AWS_MAX_ATTEMPTS # specifies a value of maximum retry attempts the AWS CLI retry handler uses +AWS_METADATA_SERVICE_NUM_ATTEMPTS # attempts to retrieve credentials once from the instance metadata service before stopping +AWS_METADATA_SERVICE_TIMEOUT # seconds before connection to instance metadata service should time out +AWS_PAGER # pager program used for output +AWS_PROFILE # name of profile with credentials and options +AWS_REGION # The AWS SDK compatible environment variable that specifies the AWS Region to send the request to +AWS_RETRY_MODE # Specifies which retry mode AWS CLI uses. There are three retry modes available: legacy (default), standard, and adaptive. +AWS_ROLE_ARN # ARN of an IAM role with a web identity provider +AWS_ROLE_SESSION_NAME # name to attach to the role session, provided to RoleSessionName parameter; used with AWS_ROLE_ARN and AWS_WEB_IDENTITY_TOKEN_FILE +AWS_SECRET_ACCESS_KEY # secret key associated with the access key +AWS_SESSION_TOKEN # session token value that is required if you are using temporary security credentials retrieved directly from aws sts +AWS_SHARED_CREDENTIALS_FILE # location of the file that the AWS CLI uses to store access keys. default path ~/.aws/credentials +AWS_STS_REGIONAL_ENDPOINTS # how to determines aws service endpoint that the cli client uses to talk to the AWS Security Token Service + # default value for version 1: "legacy" / version 2: "regional" +AWS_WEB_IDENTITY_TOKEN_FILE # path to file containing an OAuth 2.0 access token or OpenID Connect ID token +``` + +[docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) + +## option + +```sh +--color STRING # support for color output: on, off, auto +--debug # enables debug logging by providing full python logs; (`CMD 2> FILE`, `CMD &> FILE`) +--region REGION # override region +--dry-run +--output text # json +--owners amazon +--profile PROFILE +--cli-auto-prompt, --no-cli-auto-prompt +``` + +[docs.aws.amazon.com/cli/latest/userguide/cli-configure-options](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-options.html) + +## usage + +## configure + +```sh +aws configure +aws configure list +aws configure list-profiles + +aws --profile PROFILE configure set region REGION +aws --profile PROFILE configure set default.region REGION +aws --profile PROFILE configure set aws_access_key_id ACCESS_KEY_ID +aws --profile PROFILE configure set aws_secret_access_key SECRET_ACCESS_KEY +aws --profile PROFILE configure set aws_session_token AWS_SESSION_TOKEN +``` + +```sh +cat < ~/.aws/config +[profile PROFILE_NAME] +aws_cli_auto_prompt = on|on-partial +aws_cli_file_encoding = UTF-8 +aws_session_token = +ca_bundle = +cli_pager = +credential_source = Ec2InstanceMetadata +max_attempts = +output = json|yaml|yaml-stream|text|table +region = REGION +retry_mode = +role_arn = ROLE_ARN +role_session_name = +source_profile = PROFILE_NAME +web_identity_token_file = +EOF + +cat < ~/.aws/credentials +[PROFILE_NAME] +aws_access_key_id = AWS_ACCESS_KEY_ID +aws_secret_access_key = AWS_SECRET_ACCESS_KEY +EOF +``` + +[stackoverflow.com/replace-aws-keys-with-iam-role-in-aws-credentials](https://stackoverflow.com/a/50382297/14523221) + +## cloudtrail + +```sh +aws cloudtrail lookup-events --max-results 1 + +aws cloudtrail lookup-events --lookup-attributes AttributeKey=EventName,AttributeValue=DescribeVpcs +# AttributeKeys: AccessKeyId, EventId, EventName, EventSource, ReadOnly, ResourceName, ResourceType, Username +``` + +## configservice + +```sh +aws configservice select-resource-config \ + --expression "SELECT resourceType GROUP BY resourceType" | jq -r '.Results[] | fromjson | .resourceType' | sort + +aws securityhub get-findings \ + --filters \ + --sort-criteria \ + --page-size \ + --max-items +``` + +## cloudfront + +```sh +aws cloudfront list-cloud-front-origin-access-identities --output json +``` + +## cloudformation + +```sh +aws cloudformation list-stacks --query "StackSummaries[*].StackName" --stack-status-filter \ + CREATE_IN_PROGRESS \ + CREATE_COMPLETE \ + ROLLBACK_IN_PROGRESS \ + ROLLBACK_FAILED \ + ROLLBACK_COMPLETE \ + DELETE_IN_PROGRESS \ + DELETE_FAILED \ + UPDATE_IN_PROGRESS \ + UPDATE_COMPLETE_CLEANUP_IN_PROGRESS \ + UPDATE_COMPLETE \ + UPDATE_ROLLBACK_IN_PROGRESS \ + UPDATE_ROLLBACK_FAILED \ + UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS \ + UPDATE_ROLLBACK_COMPLETE \ + REVIEW_IN_PROGRESS +``` + +[[cdk]], [[cfn]], [[terraform]] + +## ec2 - elastic cloud compute + +```sh +aws ec2 describe-instances \ + --filters "Name=tag-key,Values=eks:cluster-name" "Name=tag-value,Values=eksworkshop*" \ + --query 'Reservations[*].Instances[*].[PrivateDnsName,Tags[?Key==`eks:nodegroup-name`].Value|[0],Placement.AvailabilityZone,PrivateIpAddress,PublicIpAddress]' \ + --output table + +aws ec2 describe-instances \ + --filters Name=instance-state-name,Values=running \ + --query "Reservations[*].Instances[*].InstanceId" + + --query 'Images[?CreationDate>=`2016-04-01`][]' + --query "sort_by(Images, &CreationDate)[].Name" + --query "sort_by(Images, &CreationDate)[].[Name, ImageId]" + --query 'Images[*].[ImageId,CreationDate]' + --query 'sort_by(Images,&CreationDate)[-1].ImageId' + --query 'reverse(sort_by(Images, &CreationDate))[0]' + --query 'Images[*].[ImageId,CreationDate,Architecture]' + +--filters 'Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-*-amd64-server-*' 'Name=state,Values=available' + +# latest ubuntu 20 server arm64 image => [-1:] returns the last element of the array +--filters "Name=name,Values=ubuntu*20.04-arm64-server*" \ + --query "sort_by(Images, &CreationDate)[-1:].[Name, ImageId]" + +--filters "Name=name,Values=ubuntu/images/hvm-ssd/ubuntu-xenial-16.04*" \ + --query 'sort_by(Images, &CreationDate)[-1].[CreationDate,Name,ImageId]' + +aws autoscaling describe-auto-scaling-groups \ + --query "AutoScalingGroups[? Tags[? (Key=='eks:cluster-name') && Value=='CLUSTER_NAME']].[AutoScalingGroupName, MinSize, MaxSize,DesiredCapacity]" \ + --output table + +aws autoscaling describe-auto-scaling-groups `# get auto scaling group name` \ + --query "AutoScalingGroups[? Tags[? (Key=='eks:cluster-name') && Value=='CLUSTER_NAME']].AutoScalingGroupName" \ + --output text + +aws autoscaling update-auto-scaling-group \ + --auto-scaling-group-name ASG_NAME \ + --min-size 3 --desired-capacity 3 --max-size 4 # increase/decrease capacities + + +aws ec2 create-security-group \ + --description 'RDS SG' \ + --group-name 'RDS_SG' \ + --vpc-id VPC_ID + +aws ec2 describe-subnets `# get public subnet id's used by eks clsuter`\ + --filters "Name=vpc-id,Values=VPC_ID" "Name=tag:Name,Values=EKS_CLUSTER_NAME/SubnetPublic*" \ + --query 'Subnets[*].SubnetId' \ + --output json | jq -c . + + +aws ec2 describe-instance-types --instance-types m6i.large | jq +``` + +#### find vpc dependencies when 'DependencyViolation' occure + +```sh +VPC="vpc-ID" + +aws ec2 describe-internet-gateways + --filters "Name=attachment.vpc-id,Values=$VPC" | jq -r '.InternetGateways[].InternetGatewayId' + +aws ec2 describe-subnets + --filters 'Name=vpc-id,Values='$VPC | grep SubnetId + +aws ec2 describe-route-tables + --filters 'Name=vpc-id,Values='$VPC | grep RouteTableId + +aws ec2 describe-network-acls + --filters 'Name=vpc-id,Values='$VPC | grep NetworkAclId + +aws ec2 describe-vpc-peering-connections + --filters 'Name=requester-vpc-info.vpc-id,Values='$VPC | grep VpcPeeringConnectionId + +aws ec2 describe-vpc-endpoints + --filters 'Name=vpc-id,Values='$VPC | grep VpcEndpointId + +aws ec2 describe-nat-gateways + --filters 'Name=vpc-id,Values='$VPC | grep NatGatewayId + +aws ec2 describe-security-groups + --filters 'Name=vpc-id,Values='$VPC | grep GroupId + +aws ec2 describe-instances + --filters 'Name=vpc-id,Values='$VPC | grep InstanceId + +aws ec2 describe-vpn-connections + --filters 'Name=vpc-id,Values='$VPC | grep VpnConnectionId + +aws ec2 describe-vpn-gateways + --filters 'Name=attachment.vpc-id,Values='$VPC | grep VpnGatewayId + +aws ec2 describe-network-interfaces + --filters 'Name=vpc-id,Values='$VPC | grep NetworkInterfaceId +``` + +### remove in-/egress rules from default secgroup + +```sh +SEC_GROUP_ID="(aws ec2 describe-security-groups --filter Name=vpc-id,Values=vpc-09cb7d6df2e7f2e82 Name=group-name,Values=default --query 'SecurityGroups[*].[GroupId]' --output text)" + +aws ec2 revoke-security-group-ingress --group-id $SEC_GROUP_ID \ + --ip-permissions "$(aws ec2 describe-security-groups --group-id $SEC_GROUP_ID --query "SecurityGroups[0].IpPermissions")" + +aws ec2 revoke-security-group-egress --group-id $SEC_GROUP_ID \ + --ip-permissions "$(aws ec2 describe-security-groups --group-id sg-0e6d21618a794e886 --query "SecurityGroups[0].IpPermissionsEgress")" +``` + +### instance types + +```sh +General Purpose # most popular; used for web servers, development environments +Compute Optimized # compute-intensive apps such as some scientific modeling or high-performance web servers +Memory Optimized # memory-intensive apps, such as real-time big data analytics, or running Hadoop or Spark +Accelerated Computing # additional hardware (GPUs, FPGAs) to provide massive amounts of parallel processing for tasks such as graphics processing +Storage Optimized # tasks that require huge amounts of storage, specifically with sequential read-writes, such as log processing +``` + +- [[ec2-instance-selector]] +- [[max-pods-calculator]] + +## pricing + +```sh +aws pricing describe-services --region us-east-1 | jq -r '.Services[] | .ServiceCode' +``` + +```sh +curl 'https://ec2.shop?region=us-west-2' +curl 'https://ec2.shop?region=us-west-2&filter=m4,m5,ssd' +curl 'https://ec2.shop' -H 'accept: json' +``` + +- [[ec2-instance-selector]] +- [ec2.shop](https://ec2.shop) +- [docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) + + +## elastic load balancers + +```sh +aws elb describe-load-balancers | jq '.LoadBalancerDescriptions[].LoadBalancerName' +``` + +## elastic kubernetes service + +```sh +aws eks list-clusters + +aws eks describe-cluster --name CLUSTER_NAME + +aws eks describe-cluster --output text --name CLUSTER_NAME `# get vpc id` \ + --query "cluster.resourcesVpcConfig.vpcId" \ + +aws eks get-token --cluster-name CLUSTER_NAME | jq -r '.status.token' + +aws eks update-kubeconfig --region REGION --name CLUSTER_NAME + +aws eks describe-nodegroup --nodegroup-name NODE_GROUP-202200000012 --cluster-name CLUSTER_NAME +``` + +## ecs - elastic container service + +```sh +aws ecs list-task-definitions | jq -r '.taskDefinitionArns[]' + +aws ecs describe-tasks --tasks TASK + +aws ecs put-account-setting-default \ + --name awsvpcTrunking \ + --value enabled \ + --region eu-central-1 + +aws ecs list-attributes \ + --target-type container-instance \ + --attribute-name ecs.awsvpc-trunk-id \ + --cluster base-cluster \ + --region eu-central-1 + +aws ecs execute-command --cluster CLUSTER --task TASK --container CONTAINER --interactive --command "bash" +``` + +[[ecs-cli]] + +## iam - identity and access management + +```sh +aws iam list-roles --path-prefix /aws-service-role/config.amazonaws.com/ # get roles of service-link + +aws iam list-roles | jq -r '.Roles[] | "\(.RoleName): \(.AssumeRolePolicyDocument.Statement[].Principal?)"' + +aws iam list-roles | jq '.Roles[] | {role: .RoleName, user: .AssumeRolePolicyDocument.Statement[].Principal.AWS} | select(.user != null)' + +aws iam list-policies --query 'Policies[PolicyName==`AmazonS3ReadOnlyAccess`].Arn' + +aws iam list-users | jq '.Users[].UserName' \ + | xargs -I{} aws iam list-access-keys --user-name {} | jq -r '.AccessKeyMetadata[] | "\(.UserName) \(.AccessKeyId)"' + +# aws allows assigning only one MFA device (virtual or hardware) to root accounts +# if the list-virtual-mfa-devices returns valid +aws iam list-virtual-mfa-devices \ + --assignment-status Assigned \ + --query 'VirtualMFADevices[*].SerialNumber' arn: mfa-device currently assigned is virtual, not hardware ! +# if web console won't allow removal of mfa device +aws iam delete-virtual-mfa-device --serial-number "arn:aws:iam::ACCOUNT_ID:mfa/root-account-mfa-device" + +# aws iam - access advisor +aws iam generate-service-last-accessed-details --arn ARN # returns job-id + +aws iam get-service-last-accessed-details --job-id JOB_ID + +aws iam create-policy \ + --policy-name POLICY_NAME \ + --policy-document file://~/PATH/TO/POLICY.json + + +aws iam put-group-policy \ + --group-name GROUPE_NAME \ + --policy-name POLICY_NAME \ + --policy-document POLICY_DOCUMENT + +aws iam create-user --user-name USER_NAME + +aws iam add-user-to-group --group-name GROUPE_NAME --user-name USER_NAME + +aws iam get-group --group-name GROUP_NAME + +aws iam create-access-key --user-name USER_NAME +``` + +## relational database service + +```sh +# create RDS Postgresql instance +aws rds create-db-instance \ + --db-instance-identifier RDS_NAME \ + --db-name DB_NAME \ + --db-instance-class db.t3.micro \ + --engine postgres \ + --db-subnet-group-name SEC_GROUP_NAME \ + --vpc-security-group-ids $RDS_SG \ + --master-username NAME \ + --publicly-accessible \ + --master-user-password RDS_PASSWORD \ + --backup-retention-period 0 \ + --allocated-storage 20 + +aws rds describe-db-instances `# get current status of rds e.g. creating, available,..` \ + --db-instance-identifier RDS_NAME \ + --query "DBInstances[].DBInstanceStatus" \ + --output text + +aws rds describe-db-instances `# get database endpoint` \ + --db-instance-identifier RDS_NAME \ + --query 'DBInstances[0].Endpoint.Address' \ + --output text + +aws rds delete-db-instance \ + --db-instance-identifier RDS_NAME \ + --delete-automated-backups \ + --skip-final-snapshot +``` + +## dynamodb + +```sh +aws dynamodb list-tables + +aws dynamodb scan --table-name "TABLE" \ + --filter-expression "userId = :name" \ + --expression-attribute-values '{":name":{"S":"7b13....99ea"}}' +``` + +## route53 + +``` +aws route53 list-hosted-zones | jq -r '.HostedZones[] | .Id, .Name' +``` + +## route53domains + +```sh +aws --region us-east-1 route53domains list-domains # route53domains api has no global endpoint; only works at region us-east-1 + +aws --region us-east-1 route53domains get-domain-detail --domain-name DOMAIN | jq -r '.Nameservers[].Name' +``` + +## ssm - aw services systems manager + +```sh +# get latest ecs-optimized amazon-linux 2 ami +aws ssm get-parameters --names /aws/service/ecs/optimized-ami/amazon-linux-2/recommended \ + | jq '.Parameters[].Value | fromjson' + +aws ssm get-parameters --names /aws/service/ecs/optimized-ami/amazon-linux-2/recommended \ + --region eu-central-1 --query "Parameters[0].Value" | jq 'fromjson' +``` + +## sts - security token service + +```sh +aws sts get-caller-identity --query Account --output text # get current account id + +aws sts get-access-key-info --access-key-id ACCESS_KEY_ID # returns account id for specified access key id + +aws --profile PROFILE sts get-session-token \ + --duration-seconds 900 `# 15min`\ + --serial-number arn:aws:iam::ACCOUNT:mfa/USER \ + --token-code ONETIMECODE + +# example using mfa and aws-cli +export AWS_ACCESS_KEY_ID=AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN=AWS_SESSION_TOKEN +aws sts get-session-token --serial-number "arn:aws:iam::ACC_ID:mfa/user.name" --token-code "$(op get totp ID)" # otp +``` + +## securityhub + +```sh +aws securityhub get-findings \ + --filter 'SeverityLabel={Value=CRITICAL,Comparison=EQUALS},ComplianceStatus={Value=FAILED,Comparison=EQUALS},ResourceType={Value=AwsS3Bucket,Comparison=EQUALS}' \ + | jq -r '.Findings[].Resources[].Id' + +aws securityhub get-findings \ + --filters '{"GeneratorId":[{"Value": "aws-foundational","Comparison":"PREFIX"}],"WorkflowStatus": [{"Value": "NEW","Comparison":"EQUALS"}],"Confidence": [{"Gte": 85}]}' \ + --sort-criteria '{"Field": "LastObservedAt","SortOrder": "desc"}' \ + --page-size 5 --max-items 100 + +aws securityhub get-findings \ + --filters '{"SeverityLabel":[{"Value": "HIGH","Comparison":"EQUALS"}],"WorkflowStatus": [{"Value":"NEW","Comparison":"EQUALS"}],"WorkflowStatus": [{"Value":"NOTIFIED","Comparison":"EQUALS"}]}' | jq -r '.Findings[].Resources[].Id' +``` + +## sqs - simple message queuing service + +```sh +aws sqs list-queues | jq '.QueueUrls[] | select(endswith("dlq"))' \ + | while read; do + eval aws sqs get-queue-attributes --queue-url $REPLY --attribute-names All \ + | jq -r '.Attributes | select(.ApproximateNumberOfMessages | tonumber > 0) | "\(.ApproximateNumberOfMessages) \(.QueueArn)"'; + done +``` + +## s3 - simple storage service + +```sh +aws s3 list # list buckets +aws s3 list s3://BUCKET # list objects in BUCKET +aws s3 ls s3://BUCKET --recursive +aws s3 rm s3://BUCKET --recursive + +aws s3 cp s3://BUCKET/KEY/FILE . # downlaod BUCKET object +aws s3 sync s3://BUCKET_1 . # download all objects to local dir +aws s3 sync s3://BUCKET_1 s3://BUCKET_2 --dryrun # diff two buckets + +# aws s3api +aws s3api list-buckets | jq -r '.Buckets[].Name' +aws s3api list-buckets --query 'Buckets[*].Name' + +aws s3api list-objects --bucket BUCKET | jq -r '.Contents[].Key' +aws s3api list-objects --bucket BUCKET --output text + +aws s3api head-bucket --bucket BUCKET # return error if credentials aren't valid + +aws s3api get-object-lock-configuration --bucket BUCKET --query 'ObjectLockConfiguration.ObjectLockEnabled' # check if lock config enables + + +aws s3api delete-objects --bucket BUCKET \ + --delete "$(aws s3api list-object-versions --bucket BUCKET --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" + +aws s3api delete-objects --bucket BUCKET \ + --delete "$(aws s3api list-object-versions --bucket BUCKET --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')" + +# aws s3api list-object-versions --bucket BUCKET | jq '{Objects: [.Versions[] | {Key:.Key, VersionId : .VersionId}], Quiet: false}' +# aws s3api list-object-versions --bucket BUCKET | jq -M '{Objects: [.["Versions","DeleteMarkers"][]|select(.Key == "key-value")| {Key:.Key,VersionId : .VersionId}], Quiet: false}' +``` + +## see also + +- [[cdk]] +- [[aws-nuke]] +- [[eksctl]] +- [[amazon-linux-extras]] +- [[ec2-instance-selector]] +- [[mc]] +- [[kubectl]] +- [[gcloud]] +- [[jq]], [[yq]] +- [[op]] +- [[installer]] +- [[localstack]], [[minikube]] +- [[installer]] +- [AWS Regions and Endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html) diff --git a/notes/base64.md b/notes/base64.md new file mode 100644 index 00000000..09286258 --- /dev/null +++ b/notes/base64.md @@ -0,0 +1,27 @@ +--- +tags: [linux] +title: base64 +created: '2020-09-01T08:26:05.388Z' +modified: '2022-05-17T19:21:12.184Z' +--- + +# base64 + +> encode and decode using base64 representation + +## usage + +```sh +base64 -w 0 FILE + +base64 -i IN_FILE -o OUT_FILE + +echo FOO | python -m base64 +``` + +## see also + +- [[jq]] `@base64d` +- [[echo]] +- [[openssl]] +- [[python]] diff --git a/notes/basename.md b/notes/basename.md new file mode 100644 index 00000000..59c4c70c --- /dev/null +++ b/notes/basename.md @@ -0,0 +1,22 @@ +--- +tags: [coreutils] +title: basename +created: '2019-10-04T07:16:41.100Z' +modified: '2022-04-06T11:37:26.896Z' +--- + +# basename + +> return filename or directory portion of pathname + +## usage + +```sh +basename -s .log $(ls /var/log/*.log) # get only filename stripped from extension +``` + +## see also + +- [[bash pwd]] +- [[dirname]] +- [[realpath]] diff --git a/notes/bash [[.md b/notes/bash [[.md new file mode 100644 index 00000000..b42d8dcc --- /dev/null +++ b/notes/bash [[.md @@ -0,0 +1,33 @@ +--- +tags: [shell/bash/keyword] +title: 'bash [[' +created: '2020-09-02T14:24:09.002Z' +modified: '2022-11-25T10:53:23.523Z' +--- + +# bash \[\[ + +> execute conditional command - returns a status of `0` or `1` +> EXPRESSION are composed of same primaries used by [[bash test]] + +## usage + +```sh +[[ EXPRESSION ]] + + +( EXPRESSION ) # returns value of EXPRESSION +! EXPRESSION # true if EXPRESSION is false; else false +EXPR1 && EXPR2 # true if both EXPR1 and EXPR2 are true; else false +EXPR1 || EXPR2 # true if either EXPR1 or EXPR2 is true; else false + +OPERATORS +`==`, `!=` # string to the right of operator is used as pattern and pattern matching is performed +`=~` # string to the right of operator is matched as a regular expression +`&&`, `||` # do not evaluate EXPR2 if EXPR1 is sufficient to determine the expression's value +``` + +## see also + +- [[bash test]] +- [[bash built-in vs keyword]] diff --git a/notes/bash alias.md b/notes/bash alias.md index 95ff33e4..849c20ff 100644 --- a/notes/bash alias.md +++ b/notes/bash alias.md @@ -1,10 +1,29 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash/builtin] title: bash alias created: '2019-08-02T06:42:37.554Z' -modified: '2019-08-02T06:44:43.252Z' +modified: '2022-04-06T11:38:03.695Z' --- # bash alias -alias= +## usage + +```sh +alias l='ls -l' # define alias + +alias -p # print all defined aliases in a reusable format + + +# use alias to intercept to change future invocation +ls +alias ls='/usr/bin/time ls' +ls +unalias ls +``` +## see also + +- [[bash unalias]] +- [[git log]] +- [[time]] +- [cyberciti.biz/tips/bash-aliases-mac-centos-linux-unix/](https://www.cyberciti.biz/tips/bash-aliases-mac-centos-linux-unix.html/) diff --git a/notes/bash arguments.md b/notes/bash arguments.md index cbff6bf3..8465d568 100644 --- a/notes/bash arguments.md +++ b/notes/bash arguments.md @@ -1,21 +1,15 @@ --- -tags: [bash] +tags: [shell/bash] title: bash arguments created: '2019-07-30T06:19:48.991Z' -modified: '2019-07-30T06:22:29.779Z' +modified: '2022-04-06T11:38:10.410Z' --- # bash arguments -| | | -|--|--| -| `argument` | command is split into an array of strings named arguments; `$0`command-name | -| `option` | documented type of argument modifying the behavior of a command; `-v` verbose | -| `parameter` | argument that provides information to either the command or one of its options e.g. in `-o file`, `file` is the parameter of the `-o` option | - - -[Difference between terms: "option", "argument", and "parameter"? - Stack Overflow](http://stackoverflow.com/questions/36495669/difference-between-terms-option-argument-and-parameter) -[How do I parse command line arguments in Bash? - Stack Overflow](http://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash) +- `argument` command is split into an array of strings named arguments; `$0`command-name +- `option` documented type of argument modifying the behavior of a command; `-v` verbose +- `parameter` argument that provides information to either the command or one of its options e.g. in `-o file`, `file` is the parameter of the `-o` option ## parsing flags @@ -43,8 +37,38 @@ for i in "$@"; do ;; esac done + + +CMD=(docker exec -it) +ARGS=() + +for i in "$@"; do + echo "i: $i" + case $i in + -e=*|--environment=*) + # CMD="${i#*=}" + echo "case i: ${i#*=}" + CMD+=('-e '${i#*=}) + shift # past argument=value + # shift + ;; + *) + ARGS+=($i) + ;; + esac + # shift +done + +echo ${CMD[*]} ${ARGS[*]} ``` -- [Passing named arguments to shell scripts - Unix & Linux Stack Exchange](https://unix.stackexchange.com/a/204927) -- [What is the meaning of "${1#*-}" - Stack Overflow](https://stackoverflow.com/a/41806827) +## see also + +- [Difference between terms: "option", "argument", and "parameter"?](http://stackoverflow.com/questions/36495669/difference-between-terms-option-argument-and-parameter) +- [How do I parse command line arguments in Bash?](http://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash) +- [Passing named arguments to shell scripts](https://unix.stackexchange.com/a/204927) +- [What is the meaning of "${1#*-}"](https://stackoverflow.com/a/41806827) - [Parameter expansion [Bash Hackers Wiki]](http://wiki.bash-hackers.org/syntax/pe#substring_removal) +- [[bash for]] +- [[bash case]] +- [[bash shift]] diff --git a/notes/bash arithmetic - bc - expre.md b/notes/bash arithmetic - bc - expre.md deleted file mode 100644 index 707943f9..00000000 --- a/notes/bash arithmetic - bc - expre.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -tags: [bash] -title: bash arithmetic - bc - expre -created: '2019-07-30T06:19:48.992Z' -modified: '2019-07-30T18:47:03.726Z' ---- - -# bash arithmetic - bc - expre - -> most (if not all) GNU/Linux shells only perform `integer` operations. - -- [How can I do division with variables in a Linux shell? - Stack Overflow](https://stackoverflow.com/a/18093887) -- [How to Add Calculations to a Bash Script](https://www.lifewire.com/arithmetic-in-bash-2200566) - -## printf -```sh -printf "%.3f\n" $((1+1/2)) # 1.000 - -printf "%.3f\n" $( echo "1+1/2" | bc -l ) # 1.500 -``` - - -## bc - -> `basic calculator` is an `arbitrary-precision calculator language` - -```sh -bc -l # -l, --mathlib: Uses the predefined math routines. - -echo "3.4+7/8-(5.94*3.14)" | bc # = -15.25 - -echo "2/3" | bc # = 0 - -echo "scale=2; 2/3" | bc # = .66 - -echo "(2/3)+(7/8)" | bc # = 0 - -echo "scale=2;(2/3)+(7/8)" | bc # = 1.53 - -echo "scale=4;(2/3)+(7/8)" | bc # = 1.5416 - -echo "scale=6;(2/3)+(7/8)" | bc # = 1.541666 - -echo "(2/3)+(7/8)" | bc -l # = 1.54166666666666666666 -``` - -## expr -> `evaluate expression` using only `integer` and tops out at `2^63 - 1` -```sh -expr 1 + 1 # = 2 - -myvar=$(expr 1 + 1) # $myvar == 2 - -expr $myvar + 1 # = 3 - -expr $myvar / 3 # = 1 - -expr $myvar \* 3 # = 9 -``` -[What is the difference between bcl and expr? Stack Exchange](https://unix.stackexchange.com/a/327468) - -## let - -```sh -myvar=6 - -let myvar+=1 - -echo $myvar -``` diff --git a/notes/bash arithmetic expansion.md b/notes/bash arithmetic expansion.md new file mode 100644 index 00000000..21df5931 --- /dev/null +++ b/notes/bash arithmetic expansion.md @@ -0,0 +1,48 @@ +--- +tags: [shell/bash] +title: bash arithmetic expansion +created: '2019-07-30T06:19:48.992Z' +modified: '2022-04-06T11:38:20.902Z' +--- + +# bash arithmetic expansion + +> most (if not all) GNU/Linux shells only perform `integer` operations. + +## usage + +```sh +$((EXPRESSION)) # is arithmetic expansion; return result + +((EXPRESSION)) # don't return result; e.g. variable assignment +``` + +```sh +((var=1+2)) # Simple math + +((var++)) +((var--)) +((var+=1)) +((var-=1)) + + +((var=var2*arr[2])) # Using variables + + +echo $((13380009932/1024/1024/1024)) + +echo $((13380009932/1024**3)) # 1024 to the power of 3 +``` + +## see also + +- [[bc]] +- [[expr]] +- [[bash let]] +- [[bash printf]] +- [[bash let.md]] +- [[bash for]] +- [[bash braces]] +- [How can I do division with variables ? - Stack Overflow](https://stackoverflow.com/a/18093887) +- [How to Add Calculations to a Bash Script](https://www.lifewire.com/arithmetic-in-bash-2200566) +- [What is the difference between bcl and expr? Stack Exchange](https://unix.stackexchange.com/a/327468) diff --git a/notes/bash array.md b/notes/bash array.md index c884a793..ad171930 100644 --- a/notes/bash array.md +++ b/notes/bash array.md @@ -1,12 +1,14 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash] title: bash array created: '2019-08-01T07:14:55.242Z' -modified: '2019-08-02T06:35:53.806Z' +modified: '2022-11-29T07:59:05.267Z' --- # bash array +## usage + ```sh array[0] = val # several ways to define an array array[1] = val @@ -15,8 +17,8 @@ array=([2]=val [0]=val [1]=val) array(val val val) ${array[i]} # displays array's value for this index. If no index is supplied, array element 0 is assumed -${#array[i]} # to find out the length of any element in the array -${#array[@]} # to find out how many values there are in the array +${#array[i]} # length of any element in the array +${#array[@]} # how many values there are in the array declare -A SWARM @@ -27,6 +29,54 @@ echo ${SWARM[@]} # print values echo ${!SWARM[@]} # print keys echo ${SWARM[swarm-1]} -unset SWARM[sswarm-1] +unset SWARM[swarm-1] + + +declare -a foo=(a b c) +s=1 +e=2 +foo=("${foo[@]:s:e}") # remove element +echo ${foo[@]} +b c + + +function foo() { + elem=$1; + next=$((elem+1)); + shift; + declare -a bar=($@); + len=${#bar[@]}; + [ $elem -ge $len ] && { echo "elem > len"; return 1; }; + bar=(${bar[@]:0:elem} ${bar[@]:next}); + echo "final: ${bar[@]}"; +} + + +Colors=([0]="purple" [1]="reddish-orange" [2]="light green") +echo ${Colors[@]} # purple reddish-orange light green +declare | grep Colors # Colors=([0]="purple" [1]="reddish-orange" [2]="light green") +```` + +### save line seperate output to array + +```sh +IFS_BAK="$IFS"; IFS=$'\n'; # backup $IFS for restore later +CDK_STACKS=($(npm run --silent cdk -- list)) # save output in string array, don't wrap subsheln quotes +``` + +### check array for multiple values + +```sh +echo "${CF_STACKS[*]}" | grep -f <(printf '%s\n' "${CDK_STACKS[@]}") | wc -l | xargs) + +CF_STACKS=("A" "B" "C") +CDK_STACKS=("A") +echo "${CF_STACKS[*]}" | grep -f <(printf '%s\n' "${CDK_STACKS[@]}") -c ``` -[Bash associative array examples โ€“ Andy Balaam's Blog](https://www.artificialworlds.net/blog/2012/10/17/bash-associative-array-examples/) + +## see also + +- [[bash declare]] +- [[bash mapfile]] +- [[bash readarray]] +- [Bash associative array examples โ€“ Andy Balaam's Blog](https://www.artificialworlds.net/blog/2012/10/17/bash-associative-array-examples/) diff --git a/notes/bash bats.md b/notes/bash bats.md deleted file mode 100644 index f2ecc387..00000000 --- a/notes/bash bats.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -tags: [bash] -title: bash bats -created: '2019-07-30T06:19:49.026Z' -modified: '2019-08-01T07:18:14.082Z' ---- - -# bash bats - - -## debug bats output -```sh -@test 'test-a' { - run bash -c 'echo ERROR; false' - echo "status = ${status}" - echo "output = ${output}" - [ "$status" -eq 0 ] -} -``` - -### examples - -[govmomi/test_helper.bash at master ยท vmware/govmomi ยท GitHub](https://github.com/vmware/govmomi/blob/master/govc/test/test_helper.bash) diff --git a/notes/bash bg.md b/notes/bash bg.md new file mode 100644 index 00000000..415c56bf --- /dev/null +++ b/notes/bash bg.md @@ -0,0 +1,23 @@ +--- +tags: [shell/bash/builtin] +title: bash bg +created: '2019-08-02T06:42:37.556Z' +modified: '2022-04-06T11:38:32.711Z' +--- + +# bash bg + +## usage + +```sh +bg # lists stopped or background jobs ; resume a stopped job in the background +fg # brings the most recent job in the foreground +fg # brings job to the foreground +``` + +## see also + +- [[bash jobs]] +- [[bash fg]] +- [[bash process-handling]] +- [[bash waits]] diff --git a/notes/bash bind.md b/notes/bash bind.md new file mode 100644 index 00000000..5f0b7268 --- /dev/null +++ b/notes/bash bind.md @@ -0,0 +1,43 @@ +--- +tags: [shell/bash/builtin] +title: bash bind +created: '2019-08-02T06:42:37.558Z' +modified: '2023-05-28T16:56:23.007Z' +--- + +# bash bind + +> set readline key bindings and variables / enable key macros + +## option + +```sh +-m KEYMAP # use KEYMAP as the keymap for the duration of command. keymap-names: emacs, emacs-standard, emacs-meta, emacs-ctlx, vi, vi-move, vi-command, vi-insert +-l # list names of functions +-P # list function names and bindings +-p # list functions and bindings in a form that can be reused as input +-S # list key sequences that invoke macros and their values +-s # list key sequences that invoke macros and their values in a form that can be reused as input +-V # list variable names and values +-v # list variable names and values in a form that can be reused as input +-q function-name # query about which keys invoke the named function +-u function-name # unbind all keys which are bound to the named function +-r KEYSEQ # remove the binding for KEYSEQ +-f FILENAME # read key bindings from FILENAME +-x keyseq:shell-command # cause SHELL-COMMAND to be executed when KEYSEQ is entered +-X # list key sequences bound with -x and associated commands in a form that can be reused as input +``` + +## usage + +```sh +bind -X # list key sequences bound with -x and associated commands + +bind '"\e[24~":"foobar"' # F12, prints "foobar" on cli ready for further editing +bind '"\e[24~":"pwd\n"' # keystroke to enter a command immediately add linebreak at end +``` + +## see also + +- [[gnu readline]] +- [stackoverflow.com/in-bash-how-do-i-bind-a-function-key-to-a-command](https://stackoverflow.com/questions/4200800/in-bash-how-do-i-bind-a-function-key-to-a-command) diff --git a/notes/bash braces.md b/notes/bash braces.md new file mode 100644 index 00000000..36f888bb --- /dev/null +++ b/notes/bash braces.md @@ -0,0 +1,72 @@ +--- +tags: [shell/bash] +title: bash braces +created: '2019-09-24T06:43:14.231Z' +modified: '2023-03-22T10:02:19.307Z' +--- + +# bash braces + +> `$` character introduces parameter expansion, command substitution, or arithmetic expansion + +## subshell + +```sh +$(...) # execute the command in the parens in a subshell and return its stdout + +(...) # run the commands listed in the parens in a subshell +``` + +## group + +```sh +{...} # execute the commands in the braces as a group + +COMMAND | { echo "nope"; exit 1; } +``` + +## arithmetic expansion + +```sh +$((...)) # arithmetic expansion and return the result +$((1+1)) +$((2*2)) +$((4/2)) +$((2**2)) +$((base#number)) # convert number from base to decimal +$((2#1111)) # 15 +$((8#16)) # 14 +$((16#FF)) # 255 +``` + +## parameter expansion + +``` +${...} # parameter expansion and return the value +``` + +## brace expansion + +> used to generate stings + +```sh +# consists of a sequence or a comma separated list of items inside curly braces `{}` +# sequence consists of a starting and ending-item separated by `..` +{aa,bb,cc,dd} # aa bb cc dd +{0..12} # 0 1 2 3 4 5 6 7 8 9 10 11 12 +{3..-2} # 3 2 1 0 -1 -2 +{a..g} # a b c d e f g +{g..a} # g f e d c b a +a{0..3}b # a0b a1b a2b a3b +{a,b{1..3},c} # a b1 b2 b3 c + +echo file{1,2,3}.txt +ls -l /etc/{resolv.conf, passwd} +``` + +## see also + +- [[bash arithmetic expansion]] +- [[bash parameter expansion]] +- [double-parenthesis-with-and-without-dollar - stackoverflow](https://stackoverflow.com/a/31255942/2087704) +- [linuxjournal.com/bash-brace-expansion](https://www.linuxjournal.com/content/bash-brace-expansion) diff --git a/notes/bash break.md b/notes/bash break.md new file mode 100644 index 00000000..e6db4f44 --- /dev/null +++ b/notes/bash break.md @@ -0,0 +1,36 @@ +--- +tags: [shell/bash/builtin] +title: bash break +created: '2019-08-02T06:42:37.559Z' +modified: '2022-04-27T14:19:26.096Z' +--- + +# bash break + +> Exit for, while, or until loops. + +## usage + +```sh +break n # If N is specified, break N enclosing loops + + +while [ 1 -eq 1 -a 1 -gt 0 ]; do + echo "running.."; sleep 1; + val=5 + until [ $val -eq 0 ]; do + val=$((--val)) + rand=$(( ( RANDOM % 5) + 1 )) + echo "[rand: $rand] reduced val to: $val.."; sleep 1; + if [ $val -lt $rand ]; then + break 2; + fi + done +done +``` + +## see also + +- [[bash while]] +- [[bash for]] +- [[bash until]] diff --git a/notes/bash caller.md b/notes/bash caller.md new file mode 100644 index 00000000..c218cdd9 --- /dev/null +++ b/notes/bash caller.md @@ -0,0 +1,8 @@ +--- +tags: [shell/bash/builtin] +title: bash caller +created: '2019-08-02T06:42:37.562Z' +modified: '2021-05-12T08:46:07.607Z' +--- + +# bash caller diff --git a/notes/bash case.md b/notes/bash case.md index b75b4684..c6d5176b 100644 --- a/notes/bash case.md +++ b/notes/bash case.md @@ -1,12 +1,14 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash/keyword] title: bash case created: '2019-07-30T06:19:48.993Z' -modified: '2019-08-02T06:35:53.820Z' +modified: '2021-11-16T08:24:42.397Z' --- # bash case +## usage + ```sh case expression in pattern1) @@ -20,5 +22,8 @@ case expression in ;; esac ``` -### see also: -- [bash arguments](:note:581d55d0be23e35bad7f) + +## see also + +- [[bash if]] +- [[bash arguments]] diff --git a/notes/bash cd.md b/notes/bash cd.md new file mode 100644 index 00000000..a33f6c49 --- /dev/null +++ b/notes/bash cd.md @@ -0,0 +1,27 @@ +--- +tags: [shell/bash/builtin] +title: bash cd +created: '2019-08-02T06:42:37.564Z' +modified: '2022-04-27T14:20:01.788Z' +--- + +# bash cd + +## usage + +```sh +cd # move to homedir +cd ~ # move to ~ expands to homedir `$HOME` + +cd - # change to previous dir + +shopt -s cdspell # correct misspelled dir +``` + +## see also + +- [[bash shopt]] +- [[bash dirs]] +- [[bash pushd ]] +- [[bash pwd]] +- [[bash redirects]] diff --git a/notes/bash command.md b/notes/bash command.md new file mode 100644 index 00000000..26ae27a7 --- /dev/null +++ b/notes/bash command.md @@ -0,0 +1,29 @@ +--- +tags: [shell/bash/builtin] +title: bash command +created: '2019-08-02T06:42:37.566Z' +modified: '2022-04-27T14:20:12.206Z' +--- + +# bash command + +> execute a simple command or display information about commands +suppressing shell function lookup, or display information about the specified COMMANDs + +## usage + +```sh +-p # use a default value for PATH that is guaranteed to find all of the standard utilities +-v # print a description of COMMAND similar to the `type' builtin +-V # print a more verbose description of each COMMAND +``` + +```sh +command -v CMD # posix way of checking if CMD is available +``` + +## see also + +- [[which]] +- [[bash type]] +- [[bash hash]] diff --git a/notes/bash compgen.md b/notes/bash compgen.md new file mode 100644 index 00000000..0e8c0d55 --- /dev/null +++ b/notes/bash compgen.md @@ -0,0 +1,66 @@ +--- +tags: [shell/bash/builtin] +title: bash compgen +created: '2019-08-02T06:13:41.072Z' +modified: '2022-04-27T14:20:30.288Z' +--- + +# bash compgen + +> display possible completions depending on the options + +## usage + +```sh +COMPREPLY # array variable used to store the completions +COMP_WORDS # array of all words typed after the name of the program the compspec belongs to +COMP_CWORD # index of COMP_WORDS array pointing to the word the current cursor is at + # - index of the word the cursor was when the tab key was pressed +COMP_LINE # the current command line +``` +```sh +-d # names of directory +-e # names of exported shell variables +-f # names of file and functions +-g # names of groups +-j # names of job +-s # names of service +-u # names of userAlias names +-v # names of shell variables +``` + +```sh +compgen -c # list all the commands you could run +compgen -c x # names of all commands starting with x + +compgen -a # list all the aliases you could run +compgen -a # names of alias, like alias but only alias names + +compgen -b # list all the built-ins you could run +compgen -b # names of shell builtins => list-builtins, like help + +compgen -k # list all the keywords you could run +compgen -k # names of Shell reserved words + +compgen -A function # list all the functions you could run +compgen -A function -abck # list all the above in one go +compgen -A variable | grep X # get all variables starting with X; see also: `echo ${!X*}` + + +compgen -W "now tomorrow never" +now +tomorrow +never + +compgen -W "now tomorrow never" n +now +never + +compgen -W "now tomorrow never" t +tomorrow +``` + +## see also + +- [[bash]] +- [[bash type]] diff --git a/notes/bash complete compgen.md b/notes/bash complete.md similarity index 66% rename from notes/bash complete compgen.md rename to notes/bash complete.md index 0e88a096..3142698e 100644 --- a/notes/bash complete compgen.md +++ b/notes/bash complete.md @@ -1,11 +1,13 @@ --- -tags: [bash] -title: bash complete compgen +tags: [shell/bash/builtin] +title: bash complete created: '2019-07-30T06:19:48.994Z' -modified: '2019-08-02T06:13:38.074Z' +modified: '2022-04-27T14:20:37.150Z' --- -# bash complete compgen +# bash complete + +## usage ```sh complete -W "word1 word2 .." command # -W wordlist provide a list of words for completion @@ -23,4 +25,7 @@ _dothis_completions() { complete -F _dothis_completions dothis # -F flag defining function that will provide the completions ``` -[Creating a bash completion script](https://iridakos.com/tutorials/2018/03/01/bash-programmable-completion-tutorial.html) + +## see also + +- [Creating a bash completion script](https://iridakos.com/tutorials/2018/03/01/bash-programmable-completion-tutorial.html) diff --git a/notes/bash compopt.md b/notes/bash compopt.md new file mode 100644 index 00000000..a3b42038 --- /dev/null +++ b/notes/bash compopt.md @@ -0,0 +1,8 @@ +--- +tags: [shell/bash/builtin] +title: bash compopt +created: '2019-08-02T06:42:37.573Z' +modified: '2021-05-12T08:46:07.723Z' +--- + +# bash compopt diff --git a/notes/bash continue.md b/notes/bash continue.md new file mode 100644 index 00000000..d5f3f7b5 --- /dev/null +++ b/notes/bash continue.md @@ -0,0 +1,8 @@ +--- +tags: [shell/bash/builtin] +title: bash continue +created: '2019-08-02T06:42:37.575Z' +modified: '2021-05-12T08:46:07.746Z' +--- + +# bash continue diff --git a/notes/bash database.md b/notes/bash database.md deleted file mode 100644 index 79f45e70..00000000 --- a/notes/bash database.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -tags: [bash] -title: bash database -created: '2019-07-30T06:19:48.995Z' -modified: '2019-07-30T06:22:29.785Z' ---- - -# bash database - -### simplest database -```sh -# usage: source db.sh -# then call db_set or db_get from cli - -db_set() { echo "$1,$2" >> database; } - -db_get() { grep "^$1," database | sed -e "s/^$1,//" | tail -n 1; } -``` -[Log-structured storage - Julia Evans](https://jvns.ca/blog/2017/06/11/log-structured-storage/) diff --git a/notes/bash datatypes.md b/notes/bash datatypes.md new file mode 100644 index 00000000..1697d37a --- /dev/null +++ b/notes/bash datatypes.md @@ -0,0 +1,21 @@ +--- +tags: [shell/bash] +title: bash datatypes +created: '2019-08-19T07:28:23.142Z' +modified: '2022-04-27T14:20:48.142Z' +--- + +# bash datatypes + +- string +- integer +- array +- assoc-array +- declare -A # associative array ! + +## see also + +- [[bash arithmetic]] +- [[bash test []] +- [[bash declare]] +- [[bash declare]] diff --git a/notes/bash debugging.md b/notes/bash debugging.md new file mode 100644 index 00000000..df6801ed --- /dev/null +++ b/notes/bash debugging.md @@ -0,0 +1,86 @@ +--- +tags: [shell/bash] +title: bash debugging +created: '2019-07-30T06:19:49.021Z' +modified: '2022-04-27T14:20:55.331Z' +--- + +# bash debugging + +> some ways to debug shell-scripts + +## usage + +```sh +# variables +${BASH_SOURCE} +${LINENO} +${FUNCNAME[0]} + +# flags +bash -n SCRIPT.sh # don't run commands; check for syntax errors only +set -o noexec # alternative option set inside script + +bash -v SCRIPT.sh # echo commands before running them +set -o verbose # alternative option set inside script + +bash -x SCRIPT.sh # echo commands after command-line processing +set -o xtrace # alternative option set inside script +```` + +## redirect xtrace to file + +```sh +#!/usr/bin/env bash +exec 2>>debug.log +set -x +``` + +## simple debugger + +```sh +read var # set simple breakpoint +echo -e "dbg> \c"; read cmd; eval $cmd + +$ dbg> doSomethingFunction +$ dbg> someVariable +``` + +## debugger + +```sh +echo "LINENO: $LINENO" +trap 'echo "VARIABLE-TRACE> \$variable = \"$variable\""' DEBUG # prints value of $variable after every command + +variable=29; line=$LINENO + +echo " Just initialized \$variable to $variable in line number $line." + +let "variable *= 3"; line=$LINENO + +echo " Just multiplied \$variable by 3 in line number $line." + +exit 0 + +# The "trap 'command1 . . . command2 . . .' DEBUG" construct is +#+ more appropriate in the context of a complex script, +#+ where inserting multiple "echo $variable" statements might be +#+ awkward and time-consuming. + +# Output of script: +# +# VARIABLE-TRACE> $variable = "" +# VARIABLE-TRACE> $variable = "29" +# Just initialized $variable to 29. +# VARIABLE-TRACE> $variable = "29" +# VARIABLE-TRACE> $variable = "87" +# Just multiplied $variable by 3. +# VARIABLE-TRACE> $variable = "87" +``` + +## see also + +- [[bash trap]] +- [[bash prompt]] +- [tldp.org/LDP/abs/html/debugging](http://tldp.org/LDP/abs/html/debugging.html) +- [wiki.bash-hackers.org/scripting/debuggingtips#making_xtrace_more_useful](https://wiki.bash-hackers.org/scripting/debuggingtips#making_xtrace_more_useful) diff --git a/notes/bash declare.md b/notes/bash declare.md index 7328bdbe..acb0d041 100644 --- a/notes/bash declare.md +++ b/notes/bash declare.md @@ -1,73 +1,53 @@ --- -tags: [bash] +tags: [shell/bash] title: bash declare created: '2019-07-30T06:19:48.996Z' -modified: '2019-07-30T06:22:29.786Z' +modified: '2023-06-28T08:27:14.740Z' --- # bash declare -`declare` and `typeset` are synonym +> set variable values and attributes -```sh -declare - - -r # readonly - - -i # integer - - -a # array - - -A # associative array ! - - -f # function(s) - - -x # export - - -x var=$value - -# just using the option prints all values of one type - -declare -xp # exported vairables - -declare -f # list sourced functions - -declare -F # list only function names - -declare -p # show variables - - -declare -a # variables are treated as arrays +## option -declare -A # variables are treated as associative arrays - -declare -f # uses funtion names only - -declare -F # displays function names without definitions - -declare -i # the variables are treaded as integers - -declare -r # makes the variables read-only - -declare -x # marks the variables for export via the environment +```sh +# print all values of one type +-xp # exported vairables +-f # list sourced functions +-F # list only function names +-p # show variables +-a # variables are treated as arrays + +-a # array +-A # variables are treated as associative arrays +-f # uses funtion names only +-F # displays function names without definitions +-i # the variables are treaded as integers +-r # makes the variables read-only +-x # marks the variables for export via the environment ``` -http://tldp.org/LDP/abs/html/declareref.html +## usage -## useful for identifying variables, environmental, .. ```sh -declare | grep HOME -HOME=/home/bozo - +declare -r foo=bar # readonly -zzy=68 -declare | grep zzy -zzy=68 +declare -a arr=('aa' 'bb' 'cc' 'dd' 'ee') +declare -x var=$value -Colors=([0]="purple" [1]="reddish-orange" [2]="light green") -echo ${Colors[@]} -purple reddish-orange light green -declare | grep Colors -Colors=([0]="purple" [1]="reddish-orange" [2]="light green") +# useful for identifying variables, environmental, .. +declare | grep foo # foo=bar +declare | grep Colors # Colors=([0]="purple" [1]="reddish-orange" [2]="light green") ``` + +## see also + +- [[bash export]] +- [[bash set]] +- [[bash unset]] +- [[bash typeset]] +- [[bash array]] +- [[bash readarray]] +- [declareref - tldp.org](http://tldp.org/LDP/abs/html/declareref.html) diff --git a/notes/bash dirs.md b/notes/bash dirs.md new file mode 100644 index 00000000..a2754005 --- /dev/null +++ b/notes/bash dirs.md @@ -0,0 +1,38 @@ +--- +tags: [shell/bash/builtin] +title: bash dirs +created: '2019-08-02T06:42:37.579Z' +modified: '2023-06-28T08:26:50.469Z' +--- + +# bash dirs + +> display directory stack + +## option + +```sh +-c # clear the directory stack by deleting all of the elements +-l # do not print tilde-prefixed versions of directories relative to your home directory +-p # print the directory stack with one entry per line +-v # print the directory stack with one entry per line prefixed with its position in the stack ++N # displays Nth entry counting from left of the list shown by dirs starting with zero +-N # displays Nth entry counting from right of the list shown by dirs starting with zero +``` + +## env + +```sh +DIRSTACK +``` + +## usage + +```sh +dirs +``` + +## see also + +- [[bash popd]] +- [[bash pushd]] diff --git a/notes/bash disown.md b/notes/bash disown.md new file mode 100644 index 00000000..a42f629c --- /dev/null +++ b/notes/bash disown.md @@ -0,0 +1,23 @@ +--- +tags: [shell/bash/builtin] +title: bash disown +created: '2019-08-02T06:42:37.580Z' +modified: '2022-01-17T08:05:10.070Z' +--- + +# bash disown + +> removes the process from the list of jobs + +## usage + +```sh +disown PROC_ID +disown JOB_ID +``` + +## see also + +- [[bash kill]] +- [[bash jobs]] +- [[bash process-handling]] diff --git a/notes/bash echo.md b/notes/bash echo.md new file mode 100644 index 00000000..0f0e2b01 --- /dev/null +++ b/notes/bash echo.md @@ -0,0 +1,60 @@ +--- +tags: [shell/bash/builtin] +title: bash echo +created: '2019-08-02T06:42:37.582Z' +modified: '2023-05-19T10:40:33.953Z' +--- + +# bash echo + +> write arguments to stdout + +## option + +```sh +-n # do not append a newline +-e # enable interpretation of the following backslash escapes +-E # explicitly suppress interpretation of backslash escapes +``` + +## usage + +```sh +echo -n "USER:PWD" | base64 # not linebreak ! + +echo $'\112' # print character from octal +echo $'\x4a' # print character from hex +``` + +## url encoding + +> urls can only be sent over the Internet using the ASCII character-set. + +- urls often contain characters outside the `ascii` set and has to be converted into a valid `ascii` format +- url encoding replaces unsafe `ascii` characters with a `%` followed by two hexadecimal digits +- urls cannot contain spaces - encoding normally replaces a space with a `+` sign or with `%20` + +```sh +# encode +echo -ne 'some random\nbytes' | xxd -plain | tr -d '\n' | sed 's/\(..\)/%\1/g' +curl -s -o /dev/null -w "%{url_effective}\n" --get --data-urlencode "some random" --data-urlencode "foo=bar" "" + +# decode +url_decode () +{ + : "${*//+/ }"; + echo -e "${_//%/\\x}" +} +``` + +## see also + +- [[od]] +- [[tr]] +- [[xxd]] +- [[fmt]] +- [[ascii]] +- [[base64]] +- [[bash printf]] +- [[bash variables]] +- [how-to-urlencode-data-for-curl-command](https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command) diff --git a/notes/bash emacs shortcuts.md b/notes/bash emacs shortcuts.md deleted file mode 100644 index 5f5f9e29..00000000 --- a/notes/bash emacs shortcuts.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -tags: [bash] -title: bash emacs shortcuts -created: '2019-07-30T06:19:49.019Z' -modified: '2019-08-02T06:48:01.800Z' ---- - -# bash emacs shortcuts - -| shortcut | does | -|--|--| -| CTRL+A | move to beginning of line | -| CTRL+B | moves backward one character | -| CTRL+E | moves to end of line | -| CTRL+F | moves forward one character | -| CTRL+C | halts the current command | -| CTRL+G | aborts the current editing command and ring the terminal bell | -| CTRL+D | deletes one character backward or logs out of current session, similar to exit | -| CTRL+U | kills backward from point to the beginning of line | -| CTRL+W | kills the word behind the cursor | -| CTRL+Y | retrieves (yank) last item killed | -| CTRL+J | same as RETURN | -| CTRL+K | deletes forward to end of line | -| CTRL+L | clears screen and redisplay the line | -| CTRL+M | same as RETURN | -| CTRL+O | same as RETURN, then displays next line in history file | -| CTRL+T | transposes two characters | -| CTRL+V | makes the next character typed verbatim | -| CTRL+X | lists the possible filename completefions of the current word | -| CTRL+Z | stops the current command, resume with fg in the foreground or bg in the background | -| CTRL+N | next line in command history | -| CTRL+P | previous line in command history | -| CTRL+R | searches backward | -| CTRL+S | searches forward | - -[macos - Is there a way to make alt-f and alt-b jump word forward and backward instead of printing ฦ’ and โˆซ on Mac? - Stack Overflow](https://stackoverflow.com/questions/20146972/is-there-a-way-to-make-alt-f-and-alt-b-jump-word-forward-and-backward-instead-of) diff --git a/notes/bash enable.md b/notes/bash enable.md new file mode 100644 index 00000000..a02a1ae9 --- /dev/null +++ b/notes/bash enable.md @@ -0,0 +1,30 @@ +--- +tags: [shell/bash/builtin] +title: bash enable +created: '2019-08-02T06:42:37.583Z' +modified: '2022-04-27T14:21:37.092Z' +--- + +# bash enable + +> eable and disable shell builtins + +## options + +```sh +-a print a list of builtins showing whether or not each is enabled +-n disable each NAME or display a list of disabled builtins +-p print the list of builtins in a reusable format +-s print only the names of Posix `special' builtins +``` + +## options controlling dynamic loading + +```sh +-f Load builtin NAME from shared object FILENAME +-d Remove a builtin loaded with -f +``` + +## see also + +- [[bash]] diff --git a/notes/bash environment.md b/notes/bash environment.md index a21b7010..dd5710ad 100644 --- a/notes/bash environment.md +++ b/notes/bash environment.md @@ -1,21 +1,24 @@ --- -tags: [bash] +tags: [shell/bash] title: bash environment created: '2019-07-30T06:19:48.998Z' -modified: '2019-07-30T06:22:29.787Z' +modified: '2022-04-27T14:21:42.580Z' --- # bash environment -```sh -bash -x # dump what bash runs on startup http://superuser.com/a/144777 - -env - -printenv +## usage +```sh shopt -p # show alle options ( set -o posix ; set ) | less # set shows all shell variables (exported or not); -o posix => only variables ``` -[What is the difference between 'env' and 'printenv' ?](https://unix.stackexchange.com/questions/123473/what-is-the-difference-between-env-and-printenv) + +## see also + +- [[bash shopt]] +- [[bash set]] +- [[bash debugging]] +- [[bash declare]] +- [What is the difference between 'env' and 'printenv' ?](https://unix.stackexchange.com/questions/123473/what-is-the-difference-between-env-and-printenv) diff --git a/notes/bash eval.md b/notes/bash eval.md index 7ca1fa90..c413756b 100644 --- a/notes/bash eval.md +++ b/notes/bash eval.md @@ -1,58 +1,55 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash/builtin] title: bash eval created: '2019-07-30T06:19:49.005Z' -modified: '2019-08-02T07:00:45.670Z' +modified: '2022-01-17T08:08:05.670Z' --- # bash eval >`eval` - construct command by concatenating arguments +## usage + ```sh foo='ls -lah' eval $foo -``` -```sh c="echo"; a1="Hello, "; a2="World!"; eval $c $a1 $a2 -``` -### usage - -#### Perform var assignment using var names passed as string values. -```sh +# Perform var assignment using var names passed as string values. key="mykey" val="myval" eval $key=$val echo $mykey myval -``` +# building a command array because bash handles double-quotes differently +# on command execution - this can be problematic when variables contain spaces +# e.g. --network="${NETWORKD}" +# --networkd=Docker Test +# here `Test` would be the next option instead of the argument of --network +cmd=( + docker-machine \ + create \ + --driver vmwarevsphere \ + --vmwarevsphere-datacenter=\"${ENV_VSPHERE_DATACENTER}\" \ + --vmwarevsphere-network=\"${ENV_VSPHERE_NETWORK}\" \ + ${DOCKER_HOST_FQDN} +) +eval "${cmd[@]}" -#### run cron command from file -```sh -# /etc/cron.d/repeatme -*/10 * * * * root program arg1 arg2 +# run cron command from file /etc/cron.d/repeatme +# */10 * * * * root program arg1 arg2 eval $( cut -d ' ' -f 6- /etc/cron.d/repeatme) ``` -[The perils of Bash โ€˜evalโ€™ - .debug - Medium](https://medium.com/dot-debug/the-perils-of-bash-eval-cc5f9e309cae) +## see also - -#### building create command that respects quotes ! -```sh -ENV_VSPHERE_HOSTSYSTEM='cluster/vhost01.domain.net' -ENV_VSPHERE_BOOT2DOCKER_URL='file:///Users/user/boot2docker.iso' - -CMD=( -docker-machine create ---vmwarevsphere-username=\"${ENV_VSPHERE_USERNAME}\" ---vmwarevsphere-boot2docker-url=\"${ENV_VSPHERE_BOOT2DOCKER_URL}\" -foo.node.ddev.domain.net -) - -eval ${CMD[@]} -``` +- [[cut]] +- [[crontab]] +- [[bash exec]] +- [[docker-machine]] +- [the perils of bash `eval`](https://medium.com/dot-debug/the-perils-of-bash-eval-cc5f9e309cae) diff --git a/notes/bash exec.md b/notes/bash exec.md index 33109166..bb13c80e 100644 --- a/notes/bash exec.md +++ b/notes/bash exec.md @@ -1,17 +1,69 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash/builtin] title: bash exec created: '2019-07-30T06:19:49.006Z' -modified: '2019-07-30T06:23:11.515Z' +modified: '2022-04-27T14:22:02.698Z' --- # bash exec +> execute a command that completely replaces the current process. The current shell process is destroyed, and entirely replaced by the command you specify + - replace shell with command - shell-`PID` == command-`PID` - `echo $$ == /proc/self` - spawns new proc +## usage + +```sh +-cl +-a NAME +``` + +```sh +exec FILE # redirects stdout to a file - sends all command output that would normally go to stdout to that file + +exec N > FILE # affects the entire script or current shell. Redirection in the PID of the script or shell from that point on has changed. +N > FILE # affects only the newly-forked process, not the entire script or shell + + +# causes file descriptor 3 to be opened for reading and writing on the specified tcp socket +# if command not specified, any redirections take effect in current shell and return status is 0 +exec 3<>/dev/tcp/www.google.com/80 +echo -e "GET / HTTP/1.1\r\nhost: http://www.google.com\r\nConnection: close\r\n\r\n" >&3 +cat <&3 + +``` + ```sh -exec +cat < FILE +POST /some/pathh HTTP/1.0 +User-Agent: curl/7.37.0 +Host: localhost +Accept: */* +Content-Length: 7 +Content-Type: application/x-www-form-urlencoded + +var=val +EOF + +exec 5<>/dev/tcp/www.google.com/80 +cat FILE >&5 +cat <&5 # reply ``` + +[unix.stackexchange.com/how-to-run-the-http-request-without-using-curl](https://unix.stackexchange.com/a/234089/440548) + +## see also + +- [[bash eval]] +- [[bash read]] +- [[filesystem hierarchy standard]] diff --git a/notes/bash exit.md b/notes/bash exit.md new file mode 100644 index 00000000..7a382ef5 --- /dev/null +++ b/notes/bash exit.md @@ -0,0 +1,20 @@ +--- +tags: [shell/bash/builtin] +title: bash exit +created: '2019-08-02T06:42:37.588Z' +modified: '2022-04-27T14:22:10.397Z' +--- + +# bash exit + +> exits shell with a status of N If N is omitted, the exit status is that of the last command executed + +## usage + +```sh +exit 1; +``` + +## see also + +- [[bash return]] diff --git a/notes/bash export.md b/notes/bash export.md new file mode 100644 index 00000000..304dacc1 --- /dev/null +++ b/notes/bash export.md @@ -0,0 +1,43 @@ +--- +tags: [shell/bash/builtin] +title: bash export +created: '2019-08-02T06:42:37.589Z' +modified: '2022-03-03T09:23:51.708Z' +--- + +# bash export + +> export will make variables/functions available to all shells forked from the current shell. + +## option + +```sh +-f # refer to shell functions +-n # remove the export property from each NAME +-p # display a list of all exported variables and functions +-- # disables further option processing. +``` + +## usage + +```sh +export -p # list exported variables and functions, same as `declare -x` + +export -n # remove the export property from each NAME + +export -f myFunc # pass function to any child processes + +export A=1 B=2 C=3 # mutliple vairables + +env # print exported vairables +``` + +## see also + +- [[env]] +- [[bash]] +- [[bash declare]] +- [[bash set]] +- [[bash unset]] +- [[bash declare]] +- [[heredoc]] diff --git a/notes/bash fc.md b/notes/bash fc.md index c9d172e5..93c8aeb1 100644 --- a/notes/bash fc.md +++ b/notes/bash fc.md @@ -1,15 +1,15 @@ --- -tags: [bash, bash/builtin] +tags: [shell/bash/builtin] title: bash fc created: '2019-07-30T06:19:49.048Z' -modified: '2019-08-02T06:50:07.514Z' +modified: '2023-04-15T09:33:14.623Z' --- # bash fc -`fix command` -> open last command from history in editor and run +> `fix command` - open last command from history in editor and run +## usage ```sh fc -l -1000 # list last 1000 command from history @@ -21,16 +21,19 @@ alias r='fc -s' # typing `r cc' runs the last command beginning with `cc' # and typing `r' re-executes the last command. ``` - ## running in script -if you can't source you have to set option and source history !! + ```sh #!/bin/bash -HISTFILE=~/.bash_history # Or wherever you bash history file lives -set -o history # enable history +# if you can't source you have to set option and source history !! + +HISTFILE=~/.bash_history # or wherever you bash history file lives +set -o history # enable history history | grep git ``` -["history" stops working when run inside bash script - Unix & Linux Stack Exchange](https://unix.stackexchange.com/a/112362/193945) +## see also -[Rapidly invoke an editor to write a long, complex, or tricky command](https://www.commandlinefu.com/commands/view/1446/rapidly-invoke-an-editor-to-write-a-long-complex-or-tricky-command) +- [[bash history]] +- ["history" stops working when run inside bash script - Unix & Linux Stack Exchange](https://unix.stackexchange.com/a/112362/193945) +- [Rapidly invoke an editor to write a long, complex, or tricky command](https://www.commandlinefu.com/commands/view/1446/rapidly-invoke-an-editor-to-write-a-long-complex-or-tricky-command) diff --git a/notes/bash fg.md b/notes/bash fg.md new file mode 100644 index 00000000..f934e366 --- /dev/null +++ b/notes/bash fg.md @@ -0,0 +1,39 @@ +--- +tags: [shell/bash/builtin] +title: bash fg +created: '2019-08-02T06:42:37.596Z' +modified: '2023-04-12T11:53:07.214Z' +--- + +# bash fg + +> move job to the foreground + +## usage + +```sh +%N # job number [N] +%S # invocation of job begins with string "S" +%?S # invocation of job contains within it string "S" +%% # "current" job (last job stopped in foreground or started in background) +%+ # "current" job (last job stopped in foreground or started in background) +%- # last job +$! # last background process +``` + +```sh +fg # brings a background job into the foreground +fg %+ # brings most recently invoked background job +fg %- # brings second most recently invoked background job +fg %N # brings job number N +fg %string # brings job whose command begins with string +fg %?string # brings job whose command contains string +``` + +## see also + +- [[bash jobs]] +- [[bash bg]] +- [[bash process-handling]] +- [[bash waits]] +- [[mktemp]] diff --git a/notes/bash for.md b/notes/bash for.md index cdf38aeb..622e9805 100644 --- a/notes/bash for.md +++ b/notes/bash for.md @@ -1,90 +1,57 @@ --- -tags: [bash] +tags: [shell/bash/keyword] title: bash for created: '2019-07-30T06:19:49.007Z' -modified: '2019-07-30T06:22:29.790Z' +modified: '2022-04-27T14:22:34.833Z' --- # bash for -```sh -for x := 1 to 10 do -begin - statements -end -``` +> execute commands for each member in a list + +## usage ```sh -for name [in list]; do - statements that can use $name +for name [in words]; do + command $name; done -``` -```sh f(){ for arg; do echo $arg; done } # without `in` cycle thorugh positional arguments -``` -[Bash "for" loop without a "in foo bar..." part - Unix & Linux Stack Exchange](https://unix.stackexchange.com/a/417296/193945) -#### set -```sh + +# set set 1 2 3 -for i do - echo "$i" -done -``` +for i do echo "$i"; done -#### range -> Bash v4.0+ has inbuilt support for setting up a step value using `{START..END..INCREMENT}` syntax: -```sh -for i in {0..10..2}; do - echo "Welcome $i times" -done -``` +# seq +for i in $(seq 1 2 20); do echo "count: $i"; done -#### seq -```sh -for i in $(seq 1 2 20); do - echo "Welcome $i times" -done -``` -#### three-expression c-style +# range bash-v4.0+ has inbuilt support for setting up a step value using `{START..END..INCREMENT}` syntax: +for i in {0..10..2}; do echo "count: $i"; done -```sh -for (( EXP1; EXP2; EXP3 )); do - command1 - .. -done +# three-expression c-style +for (( EXP1; EXP2; EXP3 )); do echo "test"; done for (( initialisation ; ending condition ; update )); do statements... done -for (( c=1; c<=5; c++ )); do - echo "Welcome $c times" -done -``` +for (( c=1; c<=5; c++ )); do echo "count: $c"; done -#### infinite loop -```sh -for (( ; ; )); do - echo "infinite loops [ hit CTRL+C to stop]" -done -``` +# infinite loop +for (( ; ; )); do echo "infinite loops [ hit CTRL+C to stop]"; done -#### break -```sh +# break for I in 1 2 3 4 5; do - statements1 + STATEMENT1 if (disaster-condition); then break # Abandon the loop. fi - statements3 #While good and, no disaster-condition. + STATEMENT3 #While good and, no disaster-condition. done -``` -#### continue -```sh +# continue for f in $FILES ; do if [ -f ${f}.bak ]; then # if .bak backup file exists, read next file echo "Skiping $f file..." @@ -93,3 +60,11 @@ for f in $FILES ; do cp $f $f.bak # make a backup done ``` + +## see also + +- [[xargs]] +- [[seq]] +- [[bash arithmetic expansion]] +- [[bash function]] +- [Bash "for" loop without a "in foo bar..." part - Unix & Linux Stack Exchange](https://unix.stackexchange.com/a/417296/193945) diff --git a/notes/bash function.md b/notes/bash function.md new file mode 100644 index 00000000..b931eb04 --- /dev/null +++ b/notes/bash function.md @@ -0,0 +1,62 @@ +--- +tags: [shell/bash/keyword] +title: bash function +created: '2019-07-30T06:19:49.008Z' +modified: '2022-04-27T14:22:49.998Z' +--- + +# bash function + +> keyword `function` defines a shell function + +## usage + +```sh +# the function refers to passed arguments by position (positional parameters) +# `$@` expands positional params to seperate words: `$1` `$2`..`$N` +# `$*` expands positional params to one words: `$1c$2c..` `c` is the first character of `IFS` +# `$#` holds the number of positional parameters. + +function func() { + return 0; +} + +func() ( # run function as subshell + echo 0; +) + +func(){ + for arg; do echo $arg; done # without `in` cycle thorugh positional arguments +} + + +unset -f functname # deletes a function definition + +declare -f # displays all defined functions in your login session + + +# simple bash database - usage: source db.sh +db_set() { echo "$1,$2" >> database; } +db_get() { grep "^$1," database | sed -e "s/^$1,//" | tail -n 1; } +``` + +## variables + +```sh +${FUNCNAME[0]} # Current function + +${FUNCNAME[1]} # Parent function + +${FUNCNAME[2]} # So on and so forth + +${FUNCNAME[@]} # All functions including parents +``` + +## see also + +- [[bash]] +- [[bash return]] +- [[bash local]] +- [[bash parameter expansion]] +- [arguments - What is $@ in Bash? - Stack Overflow](https://stackoverflow.com/a/3898681/2087704) +- [Log-structured storage - Julia Evans](https://jvns.ca/blog/2017/06/11/log-structured-storage/) diff --git a/notes/bash functions.md b/notes/bash functions.md deleted file mode 100644 index 15d7dfca..00000000 --- a/notes/bash functions.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -tags: [bash] -title: bash functions -created: '2019-07-30T06:19:49.008Z' -modified: '2019-07-30T06:22:29.791Z' ---- - -# bash functions - -The function refers to passed arguments by position (positional parameters). -`$@` expands positional params to seperate words: `$1` `$2`..`$N` -`$*` expands positional params to one words: `$1c$2c..` `c` is the first character of `IFS` -`$#` holds the number of positional parameters. - -[arguments - What is $@ in Bash? - Stack Overflow](https://stackoverflow.com/a/3898681/2087704) - -```sh -function func() { - .. -} -``` - -### run function as subshell -```sh -func() ( -) -``` - -```sh -unset -f functname # deletes a function definition - -declare -f # displays all defined functions in your login session -``` diff --git a/notes/bash getops.md b/notes/bash getops.md deleted file mode 100644 index bc6973d8..00000000 --- a/notes/bash getops.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -tags: [bash] -title: bash getops -created: '2019-07-30T06:19:49.009Z' -modified: '2019-07-30T06:22:29.791Z' ---- - -# bash getops - -```sh -while getopts u:d:p:f: option; do - case "${option}" in - u) - USER=${OPTARG} - ;; - d) - DATE=${OPTARG} - ;; - p) - PRODUCT=${OPTARG} - ;; - f) - FORMAT=$OPTARG - ;; - esac -done - -echo $USER -echo $DATE -echo $PRODUCT -echo $FORMAT -``` diff --git a/notes/bash getopts.md b/notes/bash getopts.md new file mode 100644 index 00000000..14558ebe --- /dev/null +++ b/notes/bash getopts.md @@ -0,0 +1,37 @@ +--- +tags: [shell/bash] +title: bash getopts +created: '2019-07-30T06:19:49.009Z' +modified: '2022-04-27T14:22:56.680Z' +--- + +# bash getopts + +> parse positional parameters as options + +## usage + +```sh +OPTIND # option-index; initialized to 1 +OPTARG # getopts places argument to OPTARG when option requires argument +OPTSTRING # contains option letters to be recognized; + # if a letter is followed by a colon, the option is expected to have an argument, which should be separated from it by white space +OPTERR # default: 1, if set 0, getopts disables the printing of error messages +``` + +```sh +while getopts u:d:p:f: option; do + case "${option}" in + u) USER=${OPTARG} ;; + d) DATE=${OPTARG} ;; + p) PRODUCT=${OPTARG} ;; + f) FORMAT=$OPTARG ;; + esac +done +``` + +## see also + +- [[bash while]] +- [[bash case]] +- [[bash arguments]] diff --git a/notes/bash globbing.md b/notes/bash globbing.md index b380bc69..2080438b 100644 --- a/notes/bash globbing.md +++ b/notes/bash globbing.md @@ -1,66 +1,63 @@ --- -tags: [bash] +tags: [shell/bash] title: bash globbing created: '2019-07-30T06:19:49.009Z' -modified: '2019-07-30T06:22:29.792Z' +modified: '2022-04-27T14:23:05.524Z' --- # bash globbing -wildcards aka `pathname-expansion` refered to as globbing. -pathname-expansion `expands`: `*`,`?`, `.` and `[..]` -```sh -# wildcards: - -# * - Matches any string, including the null string -# List all JPEG files -ls *.jpg - -# ? - Matches any single (one) character. -# List JPEG files with 1 char names (eg a.jpg, 1.jpg) -ls ?.jpg - -# [...] - Matches any one of the enclosed characters. -# Remove JPEG files that start with a capital letter -rm [A-Z]*.jpg +> wildcards aka `pathname-expansion` refered to as `globbing` +> pathname-expansion expands: `*`, `?`, `.`, `[..]` -``` +## usage ```sh man bash # some globbing explained -``` -## extended globbing -```sh -?(pattern-list) # Matches zero or one occurrence of the given patterns -*(pattern-list) # Matches zero or more occurrences of the given patterns -+(pattern-list) # Matches one or more occurrences of the given patterns -@(pattern-list) # Matches one of the given patterns -!(pattern-list) # Matches anything except one of the given patterns -``` +# wildcards +# * Matches any string, including the null string +# ? Matches any single (one) character +# [...] Matches any one of the enclosed characters -## dotglob -If set, bash includes filenames beginning with a `.` in the results of pathname expansion. +ls *.jpg # list all jpg files -```sh -shopt -s extglob # set extended globbint +ls ?.jpg # list jpg files with 1 char names (eg `a.jpg`, `1.jpg`) +rm [A-Z]*.jpg # rm jpg files that start with a capital letter + +# dotglob +# if set, bash includes filenames beginning with a `.` in the results of pathname expansion +shopt -s extglob # set extended globbint shopt -u dotglob # unset dotglob -``` -## extglob +# extended globbing - `shopt -s extglob` +# ?(pattern-list) Matches zero or one occurrence of the given patterns +# *(pattern-list) Matches zero or more occurrences of the given patterns +# +(pattern-list) Matches one or more occurrences of the given patterns +# @(pattern-list) Matches one of the given patterns +# !(pattern-list) Matches anything except one of the given patterns -If set, the extended pattern matching features described above under Pathname Expansion are enabled. -[Path name expansion - Linux Shell Scripting Tutorial - A Beginner's handbook](http://bash.cyberciti.biz/guide/Path_name_expansion) +ls +(ab|def)*+(.jpg|.gif) # list all the jpg and gif files that start with either "ab" or "def +ls ab*.jpg ab*.gif def*.jpg def*.gif # without extended globbing +ls ab+(2|3).jpg # list all files that match regex "ab(2|3)+.jpg" -```sh -echo file{1,2,3}.txt +# list all the files that aren't jpgs or GIFs +ls *!(.jpg|.gif) # wrong ".jpg" and the ".gif" of any file's name end up getting matched by the "*" and the null string at the end of the file +ls !(*.jpg|*.gif) # right -ls -l /etc/{resolv.conf, passwd} +ls !(+(ab|def)*+(.jpg|.gif)) # list all the files that aren't jpg or gif files and start with either "ab" or "def" ``` +## see also + +- [[bash shopt]] +- [[bash parameter expansion]] +- [linuxjournal.com/content/bash-extended-globbing](https://www.linuxjournal.com/content/bash-extended-globbing) +- [Path name expansion - Linux Shell Scripting Tutorial - A Beginner's handbook](http://bash.cyberciti.biz/guide/Path_name_expansion) + diff --git a/notes/bash hash.md b/notes/bash hash.md new file mode 100644 index 00000000..e2ed29f5 --- /dev/null +++ b/notes/bash hash.md @@ -0,0 +1,34 @@ +--- +tags: [linux, macos, shell/bash/builtin] +title: bash hash +created: '2019-08-02T06:42:37.599Z' +modified: '2022-04-27T14:23:33.799Z' +--- + +# bash hash + +> remember or display program locations + +- bash checks the hash table for the name to find the executable +- there is no need to put the script in your PATH, unless you want it to be available in all new shells + + +## usage + +```sh +hash # display current hash-table + +hash -l # display hash-table in format that is usable as input + +hash -p /foo.sh foo # add command with path and alias + +hash -d foo # delete remembered location + +hash -r # clear hash-tabl +``` + +## see also + +- [[bash type]] +- [[bash command]] +- [[which]] diff --git a/notes/bash help.md b/notes/bash help.md new file mode 100644 index 00000000..3a6fad17 --- /dev/null +++ b/notes/bash help.md @@ -0,0 +1,36 @@ +--- +tags: [shell/bash/builtin] +title: bash help +created: '2019-08-02T06:42:37.601Z' +modified: '2022-11-25T10:57:31.810Z' +--- + +# bash help + +> display information about builtin commands + +## option + +```sh +-d # output short description +-m # display usage in pseudo-manpage format +-s # output only a short usage synopsis for each topic matching PATTERN +``` + +## usage + +```sh +help # displays brief summaries of builtin commands + +help CMD # find out more about the function CMD + +man -k # find out more about commands not in this list +info # find out more about commands not in this list +info bash # find out more about the shell in general +``` + +## see also + +- [[man]] +- [[command-not-found]] +- [[bash \[\[]] diff --git a/notes/bash history expansion.md b/notes/bash history expansion.md index 9827188d..da5d9491 100644 --- a/notes/bash history expansion.md +++ b/notes/bash history expansion.md @@ -1,82 +1,80 @@ --- -tags: [bash] +tags: [shell/bash] title: bash history expansion created: '2019-07-30T06:19:49.010Z' -modified: '2019-07-30T06:22:29.792Z' +modified: '2022-04-27T14:27:39.423Z' --- # bash history expansion -`bang bang` +> introduce words from the history list into the input stream, making it easy to repeat commands, insert the arguments to a previous command into the current input line, or fix errors in previous commands quickly + +## usage ```sh +! # Start a history substitution, except when followed by a space, tab, the end of the line, + # โ€˜=โ€™ or โ€˜(โ€™ (when the extglob shell option is enabled using the shopt builtin). + +!! # `bang bang`: Refer to the previous command. This is a synonym for โ€˜!-1โ€™ !! # repeats the last command sudo !! -!-2 # relative to current index - +!n # Refer to command line n. !14934 # executes command in history with number 14934 -!m # executes command in history starting with "m.." +!-n # Refer to the command n lines back. +!-2 # relative to current index +!1-$:p +!string # Refer to the most recent command preceding the current position in the history list starting with string. +!m # executes command in history starting with "m.." !ech # e.g. autocompletes last command starting with ech maybe echo something -``` +!ech:p # :p prints command -### modifieres -> After the optional word designator, you can add a sequence of one or more of the following modifiers, each preceded by a `:` -```sh +!^ # get the first argument of a particular command +ls ls !^ -h # Remove a trailing pathname component, leaving only the head. -t # Remove all leading pathname components, leaving the tail. -r # Remove a trailing suffix of the form โ€˜.suffixโ€™, leaving the basename. -e # Remove all but the trailing suffix. +!$ # get the last argument +mkdir foo && cd !$ -p # Print the new command but do not execute it. -!ech:p # :p prints command +!* # get all arguments, but not the command itself -q # Quote the substituted words, escaping further substitutions. -x # Quote the substituted words as with โ€˜qโ€™, but break into words at spaces, tabs, and newlines. -s/old/new/ # Substitute new for the first occurrence of old in the event line. Any delimiter may be used in place of โ€˜/โ€™. The delimiter may be quoted in old and new with a single backslash. If โ€˜&โ€™ appears in new, it is replaced by old. A single backslash will quote the โ€˜&โ€™. The final delimiter is optional if it is the last character on the input line. -& # Repeat the previous substitution. -g -a # Cause changes to be applied over the entire event line. Used in conjunction with โ€˜sโ€™, as in gs/old/new/, or with โ€˜&โ€™. -G # Apply the following โ€˜sโ€™ modifier once to each word in the event. +!# # entire command line typed so far +!?string[?] # Refer to the most recent command preceding the current position in the history list containing string. + # trailing `?` may be omitted if the string is followed immediately by a newline + +^string1^string2^ # quick substitution - repeat the last command, replacing string1 with string2 +# Equivalent to +!!:s/string1/string2/. ``` -[Bash Reference Manual - Modifiers](https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Modifiers) +## modifieres -### Expanding for historical arguments +after optional word designator, you can add a sequence of one or more of the following modifiers, each preceded by `:` ```sh -ls !^ # get the first argument of a particular command - -!$ # get the last argument - -!1-$:p # - - -!* # get all arguments (but not the command itself) +h # remove a trailing pathname component, leaving only the head +t # remove all leading pathname components, leaving the tail +r # remove a trailing suffix of the form โ€˜.suffixโ€™, leaving the basename +e # remove all but the trailing suffix +p # print the new command but do not execute it. +q # quote the substituted words, escaping further substitutions. +x # quote the substituted words as with โ€˜qโ€™, but break into words at spaces, tabs, and newlines + +s/old/new/ # Substitute new for the first occurrence of old in the event line. Any delimiter may be used in place of `/` +# delimiter may be quoted in old and new with a single backslash. +# If โ€˜&โ€™ appears in new, it is replaced by old. +# A single backslash will quote the โ€˜&โ€™. The final delimiter is optional if it is the last character on the input line. +# & repeat the previous substitution +# g +# a cause changes to be applied over the entire event line. Used in conjunction with โ€˜sโ€™, as in gs/old/new/, or with โ€˜&โ€™ +# G apply the following โ€˜sโ€™ modifier once to each word in the event ``` -```sh -! # Start a history substitution, except when followed by a space, tab, the end of the line, - # โ€˜=โ€™ or โ€˜(โ€™ (when the extglob shell option is enabled using the shopt builtin). -!n # Refer to command line n. -!-n # Refer to the command n lines back. -!! # Refer to the previous command. This is a synonym for โ€˜!-1โ€™. -!string # Refer to the most recent command preceding the current position in the history list starting with string. - -!?string[?] # Refer to the most recent command preceding the current position in the history list containing string. - # The trailing โ€˜?โ€™ may be omitted if the string is followed immediately by a newline. - -^string1^string2^ # Quick Substitution. Repeat the last command, replacing string1 with string2. -# Equivalent to -!!:s/string1/string2/. - -!# # The entire command line typed so far. - -``` +## see also -[Bash Event-Designators](https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Event-Designators) +- [[bash history]] +- [Bash Event-Designators](https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Event-Designators) +- [Bash Reference Manual - Modifiers](https://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Modifiers) diff --git a/notes/bash history.md b/notes/bash history.md new file mode 100644 index 00000000..c2602918 --- /dev/null +++ b/notes/bash history.md @@ -0,0 +1,79 @@ +--- +tags: [shell/bash/builtin] +title: bash history +created: '2019-08-02T06:42:37.603Z' +modified: '2023-06-28T08:37:31.485Z' +--- + +# bash history + +> display or manipulate the history list + +## environment + +```sh +man bash # get available env vars + +HISTTIMEFORMAT # format-string strftime(3) to print the time stamp for each displayed history entry +HISTTIMEFORMAT='%F %T ' # => "10 2023-04-21 vi README.md" +HISTTIMEFORMAT='%F %T %t' # => "10 2023-04-21 11:50:06 vi README.md" + +HISTFILE # alternative history-file defaults to `~/.bash_history` +unset HISTFILE # don't save to history + +HISTFILESIZE=1000000 +HISTSIZE=1000000 + +HISTCONTROL +HISTCONTROL=ignorespace # start command with space to not get saved +HISTCONTROL=ignoredups # ignore duplicate liens +HISTCONTROL=ignoreboth # ignore both above + + + +HISTCONTROL # colon-separated list of values controlling how commands are saved on the history list. + # If the list of values includes ignorespace, lines which begin with a space character are not saved in the history list. A value of ignoredups causes lines matching the previous history entry to not be saved. A value of ignoreboth is shorthand for ignorespace and ignoredups. A value of erasedups causes all previous lines matching the current line to be removed from the history list before that line is saved. Any value not in the above list is ignored. If HISTCONTROL is unset, or does not include a valid value, all lines read by the shell parser are saved on the history list, subject to the value of HISTIGNORE. The second and subsequent lines of a multi-line compound command are not tested, and are added to the history regardless of the value of HISTCONTROL. +HISTFILE # filename in which command history is saved default value is ~/.bash_history. If unset history is not saved +HISTFILESIZE # max number of lines contained in the history file. When this variable is assigned a value, the history file is truncated, if necessary, to contain no more than that number of lines by removing the oldest entries. The history file is also truncated to this size after writing it when a shell exits. If the value is 0, the history file is truncated to zero size. Non-numeric values and numeric values less than zero inhibit truncation. The shell sets the default value to the value of HISTSIZE after reading any startup files. +HISTIGNORE # colon-separated list of patterns used to decide which command lines should be saved on the history list. Each pattern is anchored at the beginning of the line and must match the complete line (no implicit `*' is appended). Each pattern is tested against the line after the checks specified by HISTCONTROL are applied. In addition to the normal shell pattern matching characters, `&' matches the previous history line. `&' may be escaped using a backslash; the backslash is removed before attempting a match. The second and subsequent lines of a multi-line compound command are not tested, and are added to the history regardless of the value of HISTIGNORE. The pattern matching honors the setting of the extglob shell option. +HISTSIZE # number of commands to remember in the command history If the value is 0, commands are not saved in the history list +HISTTIMEFORMAT # If variable is set and not null, its value is used as a strftime(3) to print the time stamp associated with each history entry displayed by the history builtin +``` + +## option + +```sh +-c # clear the history list by deleting all of the entries +-d offset # delete the history entry at position OFFSET. Negative offsets count back from the end of the history list +-a # append history lines from this session to the history file +-n # read all history lines not already read from the history file and append them to the history list +-r # read the history file and append the contents to the history list +-w # write the current history to the history file +-p # perform history expansion on each ARG and display the result without storing it in the history list +-s # append the ARGs to the history list as a single entry +``` + +## usage + +```sh +shopt -s histappend # don't overwrite history file after each session +shopt -s cmdhist # force commands entered on more than one line to be adjusted to fit on only one for parsing + +PROMPT_COMMAND='history -a' + +history 1 + +# top 30 entries +history | awk '{print $2}' | sort | uniq -c | sort -rn | head -30 | \ + awk '!max{max=$1;}{r=""; i=s=100*$1/max; while(i-->0)r=r"#"; printf "%50s %5d %s %s",$2,$1,r,"\n";}' +``` + +## see also + +- [[wc]] +- [[bash fc]] +- [[bash shopt]] +- [[bash redirect]] +- [[bash history expansion]] +- [sanctum.geek.nz/arabesque/better-bash-history/](https://sanctum.geek.nz/arabesque/better-bash-history/) +- [stackoverflow.com/execute-command-without-keeping-it-in-history](https://stackoverflow.com/questions/8473121/execute-command-without-keeping-it-in-history) diff --git a/notes/bash i_o redirects.md b/notes/bash i_o redirects.md deleted file mode 100644 index 8774bb2d..00000000 --- a/notes/bash i_o redirects.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -tags: [bash] -title: bash i_o redirects -created: '2019-07-30T06:19:49.011Z' -modified: '2019-07-30T06:22:29.793Z' ---- - -# bash i/o redirects - -```sh -cmd1|cmd2 # pipe; takes standard output of cmd1 as standard input to cmd2 -> file # directs standard output to file -< file # takes standard input from file ->> file # directs standard output to file; append to file if it already exists ->|file # forces standard output to file even if noclobber is set -n>|file # forces output to file from file descriptor n even if noclobber is set -<> file # uses file as both standard input and standard output -n<>file # uses file as both input and output for file descriptor n - -<