# ---------------------------------------------------------------------- # debugging metadata and procedure # ---------------------------------------------------------------------- ### Following ### * is a record of actions done to debug this problem: https://bitbucket.org/tlroche/linode_jumpbox_config/wiki/Home#rst-header-networking-problem ### * were all done from my 'client' as defined here: https://bitbucket.org/tlroche/linode_jumpbox_config/wiki/Home#rst-header-glossary ### * were all done in a shell=bash running locally on my client ### States examined here (which were added over time) are: ### 0. OpenVPN client !running && OpenVPN server running (on separate box) ### Properties "filled out" from this template: https://bitbucket.org/tlroche/linode_jumpbox_config/raw/HEAD/scripts/private.properties_template source /path/to/project/linode_jumpbox_config/scripts/private.properties date ; pgrep -l openvpn | wc -l date ; sudo ip link show ; sudo ip addr show ; sudo ip route show # IIUC, initial routes should resemble (after checking eth0 IP# with `ip addr`) # > default via 192.168.1.1 dev eth0 proto static # > 169.254.0.0/16 dev eth0 scope link metric 1000 # > 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 # check IP# first ### 1. Starting OpenVPN client OPENVPN_CLIENT_CONF_FP="${OPENVPN_CLIENT_TARGET_DIR}/${OPENVPN_CLIENT_CONF_FN}" date ; sudo openvpn --config ${OPENVPN_CLIENT_CONF_FP} & date ; pgrep -l openvpn | wc -l ### 2. After starting OpenVPN client date ; sudo ip route show ; sudo ip link show ### 3. After logging into remote-access website (and checking whatismyip) date ; ~/bin/startF5VPN.sh # remember to login, not just start the browser/NAP! date ; sudo ip route show ; sudo ip link show ### 4. After connecting to F5VPN (requires login to remote-access website) date ; sudo ip route show ; sudo ip link show ## Note I get different IP#s for interface=ppp0 each time I run this scenario. F5VPN_DEVICE='ppp0' F5VPN_DEFAULT_GATEWAY='' # error value, will be rewritten if found (below) F5VPN_DEFAULT_DEST='0.0.0.0/1' GATEWAY_REGEXP='[0-9]+\.[0-9]+\.[0-9]+\.[0-9][^/]' # no trailing slash IP_ROUTE_SHOW="$(sudo ip route show)" if [[ -z "${IP_ROUTE_SHOW}" ]] ; then echo -e 'ERROR: IP_ROUTE_SHOW undefined, exiting ...' else F5VPN_DEFAULT_ROUTE="$(echo -e "${IP_ROUTE_SHOW}" | fgrep -e "${F5VPN_DEFAULT_DEST}")" if [[ -z "${F5VPN_DEFAULT_ROUTE}" ]] ; then echo -e 'ERROR: F5VPN_DEFAULT_ROUTE undefined, exiting ...' # elif [[ "${F5VPN_DEFAULT_ROUTE}" =~ "${GATEWAY_REGEXP}" ]] ; then # DON'T QUOTE! elif [[ ${F5VPN_DEFAULT_ROUTE} =~ ${GATEWAY_REGEXP} ]] ; then # echo -e 'Match!' F5VPN_DEFAULT_GATEWAY="${BASH_REMATCH[0]}" # echo -e "${F5VPN_DEFAULT_GATEWAY}" # TODO: parameterize command echo -e "About to do: sudo ip route del ${F5VPN_DEFAULT_DEST} via ${F5VPN_DEFAULT_GATEWAY} dev ${F5VPN_DEVICE}" sudo ip route del ${F5VPN_DEFAULT_DEST} via ${F5VPN_DEFAULT_GATEWAY} dev ${F5VPN_DEVICE} else echo -e 'ERROR: F5VPN_DEFAULT_GATEWAY not matched, exiting ...' fi # [[ -z "${F5VPN_DEFAULT_ROUTE}" ]] fi # [[ -z "${IP_ROUTE_SHOW}" ]] ### 5. After deleting F5VPN-added default route date ; sudo ip route show ; sudo ip link show ### 6. After disconnecting from F5VPN and logout from remote-access website date ; sudo ip route show ; sudo ip link show ### 7. Killing OpenVPN client date ; sudo pkill -9 openvpn date ; pgrep -l openvpn | wc -l ### 8. After killing OpenVPN client date ; sudo ip link show ; sudo ip addr show ; sudo ip route show # want to restore routes= # > default via 192.168.1.1 dev eth0 proto static # > 169.254.0.0/16 dev eth0 scope link metric 1000 # > 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 # ---------------------------------------------------------------------- # transcript follows (with some added formatting) to end of file # ---------------------------------------------------------------------- ### 0. OpenVPN client !running && OpenVPN server running (on separate box) # source /path/to/project/linode_jumpbox_config/scripts/private.properties me@client:~$ date ; pgrep -l openvpn | wc -l Fri Jan 23 15:25:22 EST 2015 0 me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:25:22 EST 2015 default via 192.168.1.1 dev eth0 proto static 169.254.0.0/16 dev eth0 scope link metric 1000 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff # IIUC, initial routes should resemble (after checking eth0 IP# with `ip addr`) # > default via 192.168.1.1 dev eth0 proto static # > 169.254.0.0/16 dev eth0 scope link metric 1000 # > 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 # check IP# first ### 1. Starting OpenVPN client me@client:~$ OPENVPN_CLIENT_CONF_FP="${OPENVPN_CLIENT_TARGET_DIR}/${OPENVPN_CLIENT_CONF_FN}" me@client:~$ date ; sudo openvpn --config ${OPENVPN_CLIENT_CONF_FP} & Fri Jan 23 15:25:51 EST 2015 [1] 21524 me@client:~$ date ; pgrep -l openvpn | wc -l Fri Jan 23 15:25:54 EST 2015 1 ### 2. After starting OpenVPN client me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:26:07 EST 2015 0.0.0.0/1 via 10.8.0.5 dev tun0 default via 192.168.1.1 dev eth0 proto static 10.8.0.1 via 10.8.0.5 dev tun0 10.8.0.5 dev tun0 proto kernel scope link src 10.8.0.6 128.0.0.0/1 via 10.8.0.5 dev tun0 169.254.0.0/16 dev eth0 scope link metric 1000 SER.VER.IP.NUM via 192.168.1.1 dev eth0 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none ### 3. After logging into remote-access website (and checking whatismyip) me@client:~$ date ; ~/bin/startF5VPN.sh Fri Jan 23 15:26:23 EST 2015 $ ls -alh /usr/local/share/firefox-3.6.28/firefox -rwxr-xr-x 1 root staff 3.9K Mar 6 2012 /usr/local/share/firefox-3.6.28/firefox $ /usr/local/share/firefox-3.6.28/firefox -d -no-remote -P 'firefox_for_F5_VPN' https://remoteaccess.epa.gov/ & ... me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:28:13 EST 2015 0.0.0.0/1 via 10.8.0.5 dev tun0 default via 192.168.1.1 dev eth0 proto static 10.8.0.1 via 10.8.0.5 dev tun0 10.8.0.5 dev tun0 proto kernel scope link src 10.8.0.6 128.0.0.0/1 via 10.8.0.5 dev tun0 169.254.0.0/16 dev eth0 scope link metric 1000 SER.VER.IP.NUM via 192.168.1.1 dev eth0 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none ### 4. After connecting to F5VPN (requires login to remote-access website) me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:29:06 EST 2015 0.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 default via 192.168.1.1 dev eth0 proto static 10.144.0.1 dev ppp0 proto kernel scope link src 10.144.1.8 128.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 134.67.15.30 via 10.8.0.5 dev tun0 proto none metric 1 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp me@client:~$ ## Note I get different IP#s for interface=ppp0 each time I run this scenario. me@client:~$ F5VPN_DEVICE='ppp0' me@client:~$ F5VPN_DEFAULT_GATEWAY='' # error value, will be rewritten if found (below) me@client:~$ F5VPN_DEFAULT_DEST='0.0.0.0/1' me@client:~$ GATEWAY_REGEXP='[0-9]+\.[0-9]+\.[0-9]+\.[0-9][^/]' # no trailing slash me@client:~$ IP_ROUTE_SHOW="$(sudo ip route show)" me@client:~$ if [[ -z "${IP_ROUTE_SHOW}" ]] ; then > echo -e 'ERROR: IP_ROUTE_SHOW undefined, exiting ...' > else > F5VPN_DEFAULT_ROUTE="$(echo -e "${IP_ROUTE_SHOW}" | fgrep -e "${F5VPN_DEFAULT_DEST}")" > if [[ -z "${F5VPN_DEFAULT_ROUTE}" ]] ; then > echo -e 'ERROR: F5VPN_DEFAULT_ROUTE undefined, exiting ...' > > # elif [[ "${F5VPN_DEFAULT_ROUTE}" =~ "${GATEWAY_REGEXP}" ]] ; then # DON'T QUOTE! > elif [[ ${F5VPN_DEFAULT_ROUTE} =~ ${GATEWAY_REGEXP} ]] ; then > # echo -e 'Match!' > F5VPN_DEFAULT_GATEWAY="${BASH_REMATCH[0]}" > # echo -e "${F5VPN_DEFAULT_GATEWAY}" > # TODO: parameterize command > echo -e "About to do: sudo ip route del ${F5VPN_DEFAULT_DEST} via ${F5VPN_DEFAULT_GATEWAY} dev ${F5VPN_DEVICE}" > sudo ip route del ${F5VPN_DEFAULT_DEST} via ${F5VPN_DEFAULT_GATEWAY} dev ${F5VPN_DEVICE} > else > echo -e 'ERROR: F5VPN_DEFAULT_GATEWAY not matched, exiting ...' > fi # [[ -z "${F5VPN_DEFAULT_ROUTE}" ]] > fi # [[ -z "${IP_ROUTE_SHOW}" ]] About to do: sudo ip route del 0.0.0.0/1 via 10.144.1.8 dev ppp0 ### 5. After deleting F5VPN-added default route me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:29:06 EST 2015 default via 192.168.1.1 dev eth0 proto static 10.144.0.1 dev ppp0 proto kernel scope link src 10.144.1.8 128.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 134.67.15.30 via 10.8.0.5 dev tun0 proto none metric 1 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp ### 6. After disconnecting from F5VPN and logout from remote-access website me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:30:30 EST 2015 ## HOW DID ROUTE=`0.0.0.0/1 via 10.144.1.8 dev ppp0` reappear! I just deleted it! 0.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 default via 192.168.1.1 dev eth0 proto static 10.144.0.1 dev ppp0 proto kernel scope link src 10.144.1.8 128.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 134.67.15.30 via 10.8.0.5 dev tun0 proto none metric 1 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp ## Note the above persist for at least a minute: me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:31:30 EST 2015 0.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 default via 192.168.1.1 dev eth0 proto static 10.144.0.1 dev ppp0 proto kernel scope link src 10.144.1.8 128.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 134.67.15.30 via 10.8.0.5 dev tun0 proto none metric 1 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 26: tun0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 100 link/none 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp ### 7. Killing OpenVPN client me@client:~$ date ; sudo pkill -9 openvpn Fri Jan 23 15:31:35 EST 2015 me@client:~$ date ; pgrep -l openvpn | wc -l Fri Jan 23 15:31:35 EST 2015 0 ### 8. After killing OpenVPN client me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:31:35 EST 2015 ## Note: zombie ppp0 routes! 0.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 default via 192.168.1.1 dev eth0 proto static 10.144.0.1 dev ppp0 proto kernel scope link src 10.144.1.8 128.0.0.0/1 via 10.144.1.8 dev ppp0 proto none metric 1 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp # want to restore routes= # > default via 192.168.1.1 dev eth0 proto static # > 169.254.0.0/16 dev eth0 scope link metric 1000 # > 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 me@client:~$ date ; sudo ip route del 0.0.0.0/1 via 10.144.1.8 dev ppp0 Fri Jan 23 15:32:40 EST 2015 me@client:~$ date ; sudo ip route del 10.144.0.1 dev ppp0 Fri Jan 23 15:32:59 EST 2015 me@client:~$ date ; sudo ip route del 128.0.0.0/1 via 10.144.1.8 dev ppp0 Fri Jan 23 15:33:16 EST 2015 me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:33:23 EST 2015 default via 192.168.1.1 dev eth0 proto static 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp me@client:~$ date ; sudo ip route add 169.254.0.0/16 dev eth0 scope link metric 1000 Fri Jan 23 15:33:53 EST 2015 me@client:~$ date ; sudo ip addr show dev eth0 Fri Jan 23 15:34:22 EST 2015 2: eth0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff inet 192.168.1.142/24 brd 192.168.1.255 scope global eth0 valid_lft forever preferred_lft forever inet6 /64 scope link valid_lft forever preferred_lft forever me@client:~$ date ; sudo ip route add 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 Fri Jan 23 15:34:45 EST 2015 me@client:~$ date ; sudo ip route show ; sudo ip link show Fri Jan 23 15:34:52 EST 2015 default via 192.168.1.1 dev eth0 proto static 169.254.0.0/16 dev eth0 scope link metric 1000 192.168.1.0/24 dev eth0 proto kernel scope link src 192.168.1.142 1: lo: mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: eth0: mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 3: wlan0: mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000 link/ether brd ff:ff:ff:ff:ff:ff 27: ppp0: mtu 1500 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3 link/ppp me@client:~$ date ; nslookup www.google.com ; date Fri Jan 23 15:35:42 EST 2015 # DNS hangs until ^C Fri Jan 23 15:35:47 EST 2015 # DNS fix me@client:~$ date ; sudo ln -sf /etc/resolvconf/run/resolv.conf /etc/resolv.conf Fri Jan 23 15:36:06 EST 2015 me@client:~$ date ; nslookup www.google.com ; date Fri Jan 23 15:36:08 EST 2015 Server: 8.8.8.8 ... Name: www.google.com Address: 64.233.171.104 Fri Jan 23 15:36:08 EST 2015