1. الرئيسية
  2. SSHsnip

SSHsnip

2017/02/12
كتابات: 
تقنية
ipqualityscore - ip lookup

xml-sitemaps - website crawler to check all public pages

Gary Sieling - Software Architect

Baeldung on Linux

Some useful SSH commands.Directory 
// Kill SSH session.
[user@server ~]# last | grep "logged in"
[user@server ~]# ps -aux | grep ssh | grep pts/1https://samaphp.com/sshsnip
[user@server ~]# kill -9 

// Count file lines from a specific folder
find web/themes/custom -name '*.js' | xargs wc -l

// Trace a specific IP and check if it is going to the right route or going to local Docker IP
tracepath 192.168.8.7 -b
// See all routes
route
// Check all defined routes
sudo ip route
// Try a specific domain in a specific route interface
curl samaphp.com --interface enp4s0
curl samaphp.com --interface tun0
// Route everything to a specific interface
sudo ip route add default dev enp4s0
// List all routes
route -n

curl --request OPTIONS "https://example.com" --insecure -v

// Getting your current Linux flavor details.
lsb_release -a

// Listing all net connections to investigate sockets
ss -r

// Showing all ports
cat /etc/services

// Review logs
// List all boots:
journalctl -b

// Get boot logs:
journalctl --since "1 hour ago" -b38d1ff38cc2e4e54ae1d90866372cb15
journalctl --since "3 days ago" -b38d1ff38cc2e4e54ae1d90866372cb15

// check all ports when ping is blocked (-Pn to skip ping checks and scan the ports, will take longer time)
nmap -p- -Pn IP_HERE
// find alive hosts in CIDR range
nmap -sn 192.168.0.1/24
// scan a list of hosts from a file
nmap -iL ./hosts.txt
// TCP SYN is a default scan (-sS) .... UDP (-sU)
// Specify the range of ports (-p) or use (-p-) for all ports not only the popular
nmap -p1-3005 IP_HERE
nmap -p22,80,443 IP_HERE

// Watching system logs
tail -f /var/log/syslog

// Analyse Apache access logs
awk '{print $4}' /var/log/apache2/access.log | cut -d: -f1 | uniq -c
// You may want to divide the total to the page requests to get a real pageviews count

// Count all IPs from access_log
awk '{ print $1 } ' /var/log/apache2/access.log |  sort | uniq | wc -l
head -n2 /var/log/apache2/access.log

// Check access logs for weird access
cat /var/log/apache2/access.log | grep '26/Jul/2022:00' | grep -v 'AppleWebKit'
cat /var/log/apache2/access.log | grep -v 'AppleWebKit' | grep Bot | grep -v SemrushBot | grep -v 403
cat /var/log/apache2/access.log | grep -v 'AppleWebKit' | grep bot | grep -v SemrushBot | grep -v DuckDuckBot | grep -v SeekportBot | grep -v 403
cat /var/log/apache2/access.log | grep ' 500 '
cat /var/log/apache2/access.log | grep '/cancel'

// check all network traffic and ports
lsof -i
lsof -i :{port}
lsof -p {process_id}
// watching network 
lsof -r 2 -i -a | grep -v 'chrome\|slack\|termius-a\|lando\|copilot-a\|postman\|DeskTime\|firefox\|notion-sn'
lsof -r 2 -i -a | grep -v 'chrome\|slack\|termius-a\|lando\|copilot-a\|postman\|DeskTime\|firefox\|notion-sn' | less --chop-long-lines +F

// Apache deny access by user agent in .htaccess

  RewriteEngine On
  RewriteCond %{HTTP_USER_AGENT} (bingbot|SemrushBot|Amazonbot|Facebot|Twitterbot|PetalBot|Googlebot|HeadlessChrome) [NC]
  RewriteRule (.*) - [F,L]


# Block critical pages

  RewriteEngine On
  RewriteCond %{HTTP_USER_AGENT} (bingbot|SemrushBot|Amazonbot|Facebot|Twitterbot|PetalBot|Googlebot|HeadlessChrome|YandexBot|AhrefsBot|DotBot|TelegramBot|DuckDuckBot|SeekportBot|mj12bot|org_bot) [NC]
  RewriteCond %{THE_REQUEST} ^(.*)\/node\s(.*)$ [OR] # /node
  RewriteCond %{THE_REQUEST} ^(.*)\/node\/(.*)$ [OR] # /node/*
  RewriteCond %{THE_REQUEST} ^(.*)\/sites(.*)$ [OR] # /sites*
  RewriteCond %{THE_REQUEST} ^(.*)\/user\/(.*)$
  RewriteRule .* - [F,L]


Bots:
"TelegramBot (like TwitterBot)" (149.154.161.199) (149.154.161.219) 
"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)" (185.191.171.24) (185.191.171.9) (185.191.171.26) (185.191.171.4) (185.191.171.33) (185.191.171.11) (185.191.171.15) (185.191.171.6) (185.191.171.12) (185.191.171.1) (185.191.171.40) (185.191.171.43) (185.191.171.3) (185.191.171.37) 
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" (77.88.5.167) (77.88.5.249) 
"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"  = (54.36.148.3) (54.36.148.190) (54.36.148.0) (54.36.149.92) (54.36.149.99) (54.36.148.83) (54.36.149.41) (54.36.148.81) (54.36.148.103) (54.36.148.135) (54.36.149.4) (54.36.148.185) (54.36.149.53) (54.36.148.133) (54.36.149.93) (54.36.149.39) (54.36.148.12) (54.36.149.61) (54.36.149.5) (54.36.148.139) (54.36.148.128) (54.36.149.21) (54.36.148.78) (54.36.149.85) (54.36.148.105) (54.36.148.203) (54.36.148.165) (54.36.148.179) (54.36.148.200) (54.36.149.103) (54.36.148.26) (54.36.149.78) (54.36.148.108) (54.36.149.44) (54.36.149.71) (54.36.148.85) (54.36.148.194) (54.36.148.192) (54.36.149.19) (54.36.149.13) (54.36.149.23) (54.36.148.249) (54.36.149.43) (54.36.148.137) (54.36.148.2)
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)" (20.191.45.212)
"'DuckDuckBot-Https/1.1; (+https://duckduckgo.com/duckduckbot)'" (20.185.79.47) 
"Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; [email protected])" (216.244.66.241)  (216.244.66.241) 
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)" = (77.88.5.167) (77.88.5.249) 
"Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)"  = (135.181.140.112)
"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)" (114.119.137.145) (114.119.137.134) (114.119.137.141) (114.119.137.146) (114.119.137.143) (114.119.137.142) 
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" (66.249.66.52) (66.249.66.59) (66.249.66.192) (66.249.66.223) (66.249.66.57)
"Googlebot-Image/1.0" (66.249.66.207) (66.249.66.55) (66.249.66.56) (GET /favicon)

"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)" (162.210.196.97) (Majestic bot)
"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots" HTTP/1.1 (52.48.145.198) (52.31.133.177) (34.253.224.215) 
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)" (63.143.42.250)
"Twitterbot/1.0" (199.16.157.183)
"Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +https://archive.org/details/archive.org_bot)" (207.241.235.151)
"Mozilla/5.0 (compatible; Let's Encrypt validation server; +https://www.letsencrypt.org)" (18.224.173.230) (3.121.110.167) (54.201.204.57) (35.158.124.146) (50.112.24.86) GET /.well-known/acme-challenge/AAAAAAA
"Chrome Privacy Preserving Prefetch Proxy" (66.249.81.140) GET /.well-known/traffic-advice

"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0"
facebookexternalhit/1.1
GuzzleHttp/7

USER_IP "WhatsApp/2.22.16.75 A"
USER_IP "WhatsApp/2.22.15.77 i"

// drush globally
ln -s /var/www/html/vendor/bin/drush /usr/bin/drush

// Log analysis
- count all access_log hits
- check 500 error details cat /var/log/apache2/access.log | grep '" 500 '
- check 403 error details cat /var/log/apache2/access.log | grep '" 500 '
- check bot hits cat /var/log/apache2/access.log | grep 'bot\|Bot'
- latest errors cat /var/log/apache2/error.log | grep grep 'error\|Error'

// Debugging SMTP connection
openssl s_client -connect smtp.gmail.com:465
openssl s_client -connect smtp.gmail.com:465 | openssl x509 -text

// docker container connect ssh
// OCI runtime exec failed: exec failed: unable to start container process: exec: "/bin/bash": stat /bin/bash: no such file or directory: unknown
// The machine might not have bash installed
docker exec -ti cc55da85b915 /bin/sh

// List all iptables rules
sudo iptables --list INPUT

// Search for long text in files, trying to detect hashes, tokens, secrets .. grep long text
grep -rEiwo '[a-z0-9]{32,100}' ./*
grep -rEwo '[a-z0-9]{32,100}' ./*

// Create new SSH user
adduser --disabled-password --gecos "" NAME
mkdir /home/NAME/.ssh
echo "KEY_HERE" > /home/NAME/.ssh/authorized_keys
usermod -aG sudo NAME
sudo sh -c 'echo "NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers'

// Show big database tables in GB for mysql
SELECT table_name AS `Table`, round(((data_length + index_length) / 1024 / 1024 / 1024), 2) `size_in_gb` FROM information_schema.TABLES WHERE table_schema = 'drupal' ORDER BY size_in_gb DESC LIMIT 10;

// Delete files that contains a specific word.
find ./config/sync -name "*.moderation_state.yml" | xargs rm

// Extract matched string between two words and print each matched string in a new line. and add string before and after 
grep -oP '(?s)(?<=btn).*?(?=outline)' file.txt | awk '{print "AA "$0"VV"}'

// Keep only lines with a specific string
sed '/msgid\|blah/!d' targeted.po >> new.po

// Delete a specific line that contains a string
sed -i '/STRING_HERE/d' config/sync/core.extension.yml

// SCP. copy from local machine to remote server using ssh
scp folder.zip [email protected]:/home/user

// Check folder sizes of current folder
du -h -d1

// Getting first line of file
echo $(head -n1 .lando.yml) > .lando.local.yml

// Add string at the end of a specific line on a file
echo $(sed -e '1s/$/__update/' .lando.local.yml) > .lando.local.yml

// Flush DNS
sudo resolvectl flush-caches
// sudo systemd-resolve --flush-caches

workstation

// tool for enabling and disabling wireless devices
rfkill list
sudo rfkill unblock Bluetooth

# Make sure your Bluetooth device has enough battery. or plug it at least into the charger.

# bluetooth monitor
sudo btmon

# if PopOS can not turn on Bluetooth switch
sudo rmmod btusb
sudo modprobe btusb

// Review cron status
systemctl status cronie
systemctl enable --now cronie.service

// Check if line exists or add it
* * * * * /usr/bin/cat ~/Desktop/2.txt | /usr/bin/grep he3llo ; [ $? -eq 0 ] && /usr/bin/echo "yes" || /usr/bin/echo 'he3llo' >> ~/Desktop/2.txt

// List all hosts
getent hosts

// Metabase mysql connection error
// No matching clause: Could not connect to address=(host=x.x.x.x)(port=3306)(type=master) : Access denied for user 'USER'@'x.x.x.x' (using password: YES) Current charset is UTF-8. If password has been set using other charset, consider using option 'passwordCharacterEncoding'
mysql --default-character-set=utf8
// and then create the user
CREATE USER USERHERE@localhost IDENTIFIED BY 'PASSWORD111';
GRANT SELECT, SHOW VIEW ON drupal.* TO USERHERE@localhost IDENTIFIED BY 'PASSWORD111';
GRANT SELECT, SHOW VIEW ON drupal.* TO USERHERE@'x.x.x.x' IDENTIFIED BY 'PASSWORD111';

// Apache install multiple php versions
add-apt-repository -y ppa:ondrej/php
apt install software-properties-common
apt install php8.1 libapache2-mod-php8.1
a2enmod proxy_fcgi setenvif
apt install php8.1-fpm libapache2-mod-fcgid
a2enconf php8.1-fpm
systemctl restart apache2
systemctl status php8.1-fpm

// Add this inside VirtualHost tag in the apache .conf file of the targeted site


    # Enable http authorization headers
    
        SetEnvIfNoCase ^Authorization$ "(.+)" HTTP_AUTHORIZATION=$1
    

    
        SetHandler "proxy:unix:/run/php/php8.2-fpm.sock|fcgi://php82.localhost"
    
   # Deny access to files without filename (e.g. '.php')
    
        Require all denied
    



// To check maximum RAM your motherboard can support
sudo apt install dmidecode
sudo dmidecode -t 16

// Show CPU info
cat /proc/cpuinfo

// Docker composer
docker-compose -p lab-mailhog up -d

// Enable Bluetooth automatically on login
sudo nano /etc/bluetooth/main.conf
// Scroll down to the bottom, where you will see this: #AutoEnable=false and enable it and change it to true

// Lando error running Traefik proxy, custom proxy domains not working issue (Network error when visiting a proxy domain)
```
--2022-09-20 11:28:46--  http://dl-cdn.alpinelinux.org/alpine/v3.11/community/x86_64/APKINDEX.tar.gz
Resolving dl-cdn.alpinelinux.org (dl-cdn.alpinelinux.org)... 199.232.82.133, 2a04:4e42:54::645
Connecting to dl-cdn.alpinelinux.org (dl-cdn.alpinelinux.org)|199.232.82.133|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 857646 (838K) [application/octet-stream]
Saving to: ‘APKINDEX.tar.gz’
APKINDEX.tar.gz                                    0%[                                                                                                        ]       0  --.-KB/s    in 0s      
2022-09-20 11:28:47 (0.00 B/s) - Read error at byte 0/857646 (Connection reset by peer). Retrying.
```
The solution is to making sure the network is allowing these URLs:
http://dl-cdn.alpinelinux.org/alpine/v3.11/community/x86_64/APKINDEX.tar.gz
http://dl-cdn.alpinelinux.org/alpine/v3.11/main/x86_64/APKINDEX.tar.gz

// zsh p10k config
nano ~/.p10k.zsh

## Add these into the end of this file
## Options section
setopt correct                                                  # Auto correct mistakes
setopt extendedglob                                             # Extended globbing. Allows using regular expressions with *
setopt nocaseglob                                               # Case insensitive globbing
setopt rcexpandparam                                            # Array expension with parameters
setopt nocheckjobs                                              # Don't warn about running processes when exiting
setopt numericglobsort                                          # Sort filenames numerically when it makes sense
setopt nobeep                                                   # No beep
setopt appendhistory                                            # Immediately append history instead of overwriting
setopt histignorealldups                                        # If a new command is a duplicate, remove the older one
setopt autocd                                                   # if only directory path is entered, cd there.
setopt inc_append_history                                       # save commands are added to the history immediately, otherwise only when shell exits.
setopt histignorespace                                          # Don't save commands that start with space

HISTFILE=~/.zhistory
HISTSIZE=10000
SAVEHIST=10000

// history is showing only a few lines .. last 20 lines
// to show all history lines:
history 1
history -50

alias history='history -50'

// zsh history is reset
HISTFILE=~/.zhistory
HISTSIZE=10000
SAVEHIST=10000

// Remove text after the space on every line in the file.
cut -f1 -d' ' list.txt > list.txt
// https://text-compare.com/

// Error
// mod_fcgid: HTTP request length 138570 (so far) exceeds MaxRequestLen
nano /home/project/conf/web/DOMAIN.httpd.ssl.conf
// Add this block

FcgidMaxRequestLen 2000000

// Before 
// then
service httpd reload

// Extract git changes in one line
git status -s

// Getting remote URL
git config --get remote.origin.url

// Showing git directory
echo $(git rev-parse --show-toplevel)

// ERROR 1273 (HY000) at line 25: Unknown collation: 'utf8mb4_0900_ai_ci'
sed -i 's/utf8mb4_0900_ai_ci/utf8mb4_general_ci/g' DB.sql

// Comment out all lines after a specific text `win` word from README file
awk '/^win/{f=1}f{$0 = "#" $0}{print}' README.md

// Add string before a specific text (add `GG` as a new line before `ff` line)
awk '/ff/{print "GG"}1' README.md

// Add PHP 8.2 template for VestaCP
awk '/\/VirtualHost/{print "\n\n    \n        SetEnvIfNoCase ^Authorization$ \"(.+)\" HTTP_AUTHORIZATION=$1\n    \n\n    \n        SetHandler \"proxy:unix:/run/php/php8.2-fpm.sock|fcgi://php82.localhost\"\n    \n    \n        Require all denied\n    \n\n"}1' /usr/local/vesta/data/templates/web/apache2/default.stpl >> /usr/local/vesta/data/templates/web/apache2/php82.stpl
awk '/\/VirtualHost/{print "\n\n    \n        SetEnvIfNoCase ^Authorization$ \"(.+)\" HTTP_AUTHORIZATION=$1\n    \n\n    \n        SetHandler \"proxy:unix:/run/php/php8.2-fpm.sock|fcgi://php82.localhost\"\n    \n    \n        Require all denied\n    \n\n"}1' /usr/local/vesta/data/templates/web/apache2/default.tpl >> /usr/local/vesta/data/templates/web/apache2/php82.tpl

// replace the whole line if has word command line
// use sed with pipe
awk '/ff/{print "GG"}1' README.md | sed '/aa/c\AA'

README file content is:
```
aa this word
bb
ff
```

// install nodejs
curl -sL https://deb.nodesource.com/setup_18.x | sudo -E bash -

// You may want to install ca-certificates package
sudo apt-get install -y ca-certificates

// Trust a self signed certificate
1. Copy your .crt file to dir /usr/local/share/ca-certificates/
1. Update the CA store: `sudo update-ca-certificates`

// Apache Block IP from htaccess or vhost file
<Directory /var/www/html/>
  ORDER ALLOW,DENY
  DENY FROM 111.111.111.111
  ALLOW FROM ALL
</Directory>

// Export query results into CSV file
SELECT * FROM users
INTO OUTFILE '/var/lib/mysql-files/data.csv'

// Gitlab runner CI CD error
// Reinitialized existing Git repository in ...
// remote: You are not allowed to download code from this project
// fatal: unable to access .....git The requested URL returned error: 403
SOLUTION:
changing the repo to internal and setting project privacy settings to only project members

Executing "step_script" stage of the job script
..... deployfilename.sh: command not found
make sure this shell script is executable chmod +x

// Lando error when start
ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network

// SOLUTION: remove all networks not used by at least one container
docker network prune

// Display all existing connections
netstat- a  

// TCP connections
netstat -at  

// UDP connections
netstat -au

// only Listening Connections
netstat -tnl

-p to show pid/program name

// Using LDAP credentials for WPA2 WiFi networks
Security: WPA & WPA2, Enterprise
Authentication : Tunneled TLS
Anonymous identity: ...... keep it empty
Domain: ...... keep it empty
CHECKED: No CA certificate is required
Inner authentication: MSCHAPv2 (no EAP)
Username: YOUR_USER_NAME_HERE
Password: YOUR_PASSWORD

// Apache rules to block specific internal path and allow specific ips
// This rule might not work if you put it at the end of file. make sure it is on the first lines of htaccess file

    RewriteEngine On
    #RewriteCond %{REMOTE_ADDR} !=10.10.10.10
    # allow ip range
    RewriteCond %{REMOTE_ADDR} !^10\.10\..*$
    # allow specific ip using X-FORWARDED-FOR
    #RewriteCond %{HTTP:X-FORWARDED-FOR} !^10\.10\.10\.10$
    RewriteRule ^telescope - [F,L]


// List open ports
ss -l -p -n

// Allow port
sudo ufw allow in 32771/tcp

// Remove any line that contains a specific string or text
grep -rl 'rabbit' config/sync/* | xargs sed -i '/rabbit/d'

// Update git server IP for server deploy user. (You need this if the git server has been moved to a new IP)
echo '192.168.0.14 git.local' >> /etc/hosts
ssh-keygen -f "/home/deploy/.ssh/known_hosts" -R "git.local"
ssh-keyscan -H git.local >> /home/deploy/.ssh/known_hosts

// SNIPPET SSH capture port 80
// First install tcpflow from Ubuntu official repositories:
sudo apt-get install tcpflow
// Then run this command to inspect all HTTP requests on standard port:
sudo tcpflow -p -c port 80
// https://askubuntu.com/a/654993
sudo tcpdump dst port 80

// Use Apache as a proxy for node js app nodejs
< IfModule mod_ssl.c>
< VirtualHost *:443>
  ServerName mydomain.local
  ServerAdmin [email protected]

  ProxyPass / http://localhost:4000/
  ProxyPassReverse / http://localhost:4000/

  SSLCertificateFile /etc/certs/mydomain.local.crt
  SSLCertificateKeyFile /etc/certs/mydomain.local.key
  SSLCertificateChainFile /etc/certs/mydomain.local.pem
< /VirtualHost>
< /IfModule>
< VirtualHost *:80>
  ServerName mydomain.local
  Redirect / https://mydomain.local
< /VirtualHost>

// Show MySQL process list
while [ true ]; do mysql --execute='SHOW FULL processlist;'; sleep 1; done;

// check mysql db sizes
du -hS -d1 /var/lib/mysql

فضلاً إذا أعجبتك هذه الصفحة لاتنسى أن تقوم بمشاركتها