Patchwork [7/9] oe-git-proxy: Add a new comprehensive git proxy script

login
register
mail settings
Submitter Darren Hart
Date Feb. 5, 2013, 10:52 p.m.
Message ID <8bf58d16654191479ceb9e3863b9f0d9ae629af7.1360104547.git.dvhart@linux.intel.com>
Download mbox | patch
Permalink /patch/44179/
State New
Headers show

Comments

Darren Hart - Feb. 5, 2013, 10:52 p.m.
oe-git-proxy.sh is a simple tool to be used via GIT_PROXY_COMMAND. It
uses BSD netcat to make SOCKS5 or HTTPS proxy connections. It uses
ALL_PROXY to determine the proxy server, protocol, and port. It uses
NO_PROXY to skip using the proxy for a comma delimited list of hosts,
host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It is
known to work with both bash and dash shells.

V2: Implement recommendations by Enrico Scholz:
    o Use exec for the nc calls
    o Use "$@" instead of $* to avoid quoting issues inherent with $*
    o Use bash explicitly and simplify some of the string manipulations
    Also:
    o Drop the .sh in the name per Otavio Salvador
    o Remove a stray debug statement

Signed-off-by: Darren Hart <dvhart@linux.intel.com>
Cc: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
Cc: Otavio Salvador <otavio@ossystems.com.br>
---
 scripts/oe-git-proxy |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 124 insertions(+), 0 deletions(-)
 create mode 100755 scripts/oe-git-proxy
Otavio Salvador - Feb. 5, 2013, 11:51 p.m.
On Tue, Feb 5, 2013 at 8:52 PM, Darren Hart <dvhart@linux.intel.com> wrote:
> oe-git-proxy.sh is a simple tool to be used via GIT_PROXY_COMMAND. It
> uses BSD netcat to make SOCKS5 or HTTPS proxy connections. It uses
> ALL_PROXY to determine the proxy server, protocol, and port. It uses
> NO_PROXY to skip using the proxy for a comma delimited list of hosts,
> host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It is
> known to work with both bash and dash shells.
>
> V2: Implement recommendations by Enrico Scholz:
>     o Use exec for the nc calls
>     o Use "$@" instead of $* to avoid quoting issues inherent with $*
>     o Use bash explicitly and simplify some of the string manipulations
>     Also:
>     o Drop the .sh in the name per Otavio Salvador
>     o Remove a stray debug statement
>
> Signed-off-by: Darren Hart <dvhart@linux.intel.com>
> Cc: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
> Cc: Otavio Salvador <otavio@ossystems.com.br>
> ---
>  scripts/oe-git-proxy |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 124 insertions(+), 0 deletions(-)
>  create mode 100755 scripts/oe-git-proxy
>
> diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy
> new file mode 100755
> index 0000000..4f1871a
> --- /dev/null
> +++ b/scripts/oe-git-proxy
> @@ -0,0 +1,124 @@
> +#!/bin/bash
> +
> +# oe-git-proxy.sh is a simple tool to be via GIT_PROXY_COMMAND. It uses BSD netcat

Please fix the script name.

> +# to make SOCKS5 or HTTPS proxy connections. It uses ALL_PROXY to determine the
> +# proxy server, protocol, and port. It uses NO_PROXY to skip using the proxy for
> +# a comma delimited list of hosts, host globs (*.example.com), IPs, or CIDR masks
> +# (192.168.1.0/24). It is known to work with both bash and dash shells.
> +#
> +# BSD netcat is provided by netcat-openbsd on Ubuntu and nc on Fedora.
> +#
> +# Example ALL_PROXY values:
> +# ALL_PROXY=socks://socks.example.com:1080
> +# ALL_PROXY=https://proxy.example.com:8080
> +#
> +# Copyright (c) 2013, Intel Corporation.
> +# All rights reserved.

And please add a clear license here.

> +# AUTHORS
> +# Darren Hart <dvhart@linux.intel.com>
> +
> +# Locate the netcat binary
> +NC=$(which nc 2>/dev/null)
> +if [ $? -ne 0 ]; then
> +       echo "ERROR: nc binary not in PATH"
> +       exit 1
> +fi
> +METHOD=""
> +
> +# Test for a valid IPV4 quad with optional bitmask
> +valid_ipv4() {
> +       echo $1 | egrep -q "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}(/(3[0-2]|[1-2]?[0-9]))?$"
> +       return $?
> +}
> +
> +# Convert an IPV4 address into a 32bit integer
> +ipv4_val() {
> +       IP="$1"
> +       SHIFT=24
> +       VAL=0
> +       for B in ${IP//./ }; do
> +               VAL=$(($VAL+$(($B<<$SHIFT))))
> +               SHIFT=$(($SHIFT-8))
> +       done
> +       echo "$VAL"
> +}
> +
> +# Determine if two IPs are equivalent, or if the CIDR contains the IP
> +match_ipv4() {
> +       CIDR=$1
> +       IP=$2
> +
> +       if [ -z "${IP%%$CIDR}" ]; then
> +               return 0
> +       fi
> +
> +       # Determine the mask bitlength
> +       BITS=${CIDR##*/}
> +       if [ -z "$BITS" ]; then
> +               return 1
> +       fi
> +
> +       IPVAL=$(ipv4_val $IP)
> +       IP2VAL=$(ipv4_val ${CIDR%%/*})
> +
> +       # OR in the unmasked bits
> +       for i in $(seq 0 $((32-$BITS))); do
> +               IP2VAL=$(($IP2VAL|$((1<<$i))))
> +               IPVAL=$(($IPVAL|$((1<<$i))))
> +       done
> +
> +       if [ $IPVAL -eq $IP2VAL ]; then
> +               return 0
> +       fi
> +       return 1
> +}
> +
> +# Test to see if GLOB matches HOST
> +match_host() {
> +       HOST=$1
> +       GLOB=$2
> +
> +       if [ -z "${HOST%%$GLOB}" ]; then
> +               return 0
> +       fi
> +
> +       # Match by netmask
> +       if valid_ipv4 $GLOB; then
> +               HOST_IP=$(gethostip -d $HOST)
> +               if valid_ipv4 $HOST_IP; then
> +                       match_ipv4 $GLOB $HOST_IP
> +                       if [ $? -eq 0 ]; then
> +                               return 0
> +                       fi
> +               fi
> +       fi
> +
> +       return 1
> +}
> +
> +# If no proxy is set, just connect directly
> +if [ -z "$ALL_PROXY" ]; then
> +       exec $NC -X connect "$@"
> +fi
> +
> +# Connect directly to hosts in NO_PROXY
> +for H in ${NO_PROXY//,/ }; do
> +       if match_host $1 $H; then
> +               METHOD="-X connect"
> +               break
> +       fi
> +done
> +
> +if [ -z "$METHOD" ]; then
> +       # strip the protocol and the trailing slash
> +       PROTO=$(echo $ALL_PROXY | sed -e 's/\([^:]*\):\/\/.*/\1/')
> +       PROXY=$(echo $ALL_PROXY | sed -e 's/.*:\/\/\([^:]*:[0-9]*\).*/\1/')
> +       if [ "$PROTO" = "socks" ]; then
> +               METHOD="-X 5 -x $PROXY"
> +       elif [ "$PROTO" = "https" ]; then
> +               METHOD="-X connect -x $PROXY"
> +       fi
> +fi
> +
> +exec $NC $METHOD "$@"
> --
> 1.7.5.4
>



--
Otavio Salvador                             O.S. Systems
E-mail: otavio@ossystems.com.br  http://www.ossystems.com.br
Mobile: +55 53 9981-7854              http://projetos.ossystems.com.br
Darren Hart - Feb. 6, 2013, 12:04 a.m.
On 02/05/2013 03:51 PM, Otavio Salvador wrote:
> On Tue, Feb 5, 2013 at 8:52 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>> oe-git-proxy.sh is a simple tool to be used via GIT_PROXY_COMMAND. It
>> uses BSD netcat to make SOCKS5 or HTTPS proxy connections. It uses
>> ALL_PROXY to determine the proxy server, protocol, and port. It uses
>> NO_PROXY to skip using the proxy for a comma delimited list of hosts,
>> host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It is
>> known to work with both bash and dash shells.
>>
>> V2: Implement recommendations by Enrico Scholz:
>>     o Use exec for the nc calls
>>     o Use "$@" instead of $* to avoid quoting issues inherent with $*
>>     o Use bash explicitly and simplify some of the string manipulations
>>     Also:
>>     o Drop the .sh in the name per Otavio Salvador
>>     o Remove a stray debug statement
>>
>> Signed-off-by: Darren Hart <dvhart@linux.intel.com>
>> Cc: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
>> Cc: Otavio Salvador <otavio@ossystems.com.br>
>> ---
>>  scripts/oe-git-proxy |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++
>>  1 files changed, 124 insertions(+), 0 deletions(-)
>>  create mode 100755 scripts/oe-git-proxy
>>
>> diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy
>> new file mode 100755
>> index 0000000..4f1871a
>> --- /dev/null
>> +++ b/scripts/oe-git-proxy
>> @@ -0,0 +1,124 @@
>> +#!/bin/bash
>> +
>> +# oe-git-proxy.sh is a simple tool to be via GIT_PROXY_COMMAND. It uses BSD netcat
> 
> Please fix the script name.
> 
>> +# to make SOCKS5 or HTTPS proxy connections. It uses ALL_PROXY to determine the
>> +# proxy server, protocol, and port. It uses NO_PROXY to skip using the proxy for
>> +# a comma delimited list of hosts, host globs (*.example.com), IPs, or CIDR masks
>> +# (192.168.1.0/24). It is known to work with both bash and dash shells.
>> +#
>> +# BSD netcat is provided by netcat-openbsd on Ubuntu and nc on Fedora.
>> +#
>> +# Example ALL_PROXY values:
>> +# ALL_PROXY=socks://socks.example.com:1080
>> +# ALL_PROXY=https://proxy.example.com:8080
>> +#
>> +# Copyright (c) 2013, Intel Corporation.
>> +# All rights reserved.
> 
> And please add a clear license here.


Both fixed in the contrib branch. GPL license blurb added.

Have you happened to have tested the script?

Thanks Otavio.
Otavio Salvador - Feb. 6, 2013, 12:09 a.m.
On Tue, Feb 5, 2013 at 10:04 PM, Darren Hart <dvhart@linux.intel.com> wrote:
> On 02/05/2013 03:51 PM, Otavio Salvador wrote:
>> On Tue, Feb 5, 2013 at 8:52 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>>> oe-git-proxy.sh is a simple tool to be used via GIT_PROXY_COMMAND. It
>>> uses BSD netcat to make SOCKS5 or HTTPS proxy connections. It uses
>>> ALL_PROXY to determine the proxy server, protocol, and port. It uses
>>> NO_PROXY to skip using the proxy for a comma delimited list of hosts,
>>> host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It is
>>> known to work with both bash and dash shells.
>>>
>>> V2: Implement recommendations by Enrico Scholz:
>>>     o Use exec for the nc calls
>>>     o Use "$@" instead of $* to avoid quoting issues inherent with $*
>>>     o Use bash explicitly and simplify some of the string manipulations
>>>     Also:
>>>     o Drop the .sh in the name per Otavio Salvador
>>>     o Remove a stray debug statement
>>>
>>> Signed-off-by: Darren Hart <dvhart@linux.intel.com>
>>> Cc: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
>>> Cc: Otavio Salvador <otavio@ossystems.com.br>
>>> ---
>>>  scripts/oe-git-proxy |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++
>>>  1 files changed, 124 insertions(+), 0 deletions(-)
>>>  create mode 100755 scripts/oe-git-proxy
>>>
>>> diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy
>>> new file mode 100755
>>> index 0000000..4f1871a
>>> --- /dev/null
>>> +++ b/scripts/oe-git-proxy
>>> @@ -0,0 +1,124 @@
>>> +#!/bin/bash
>>> +
>>> +# oe-git-proxy.sh is a simple tool to be via GIT_PROXY_COMMAND. It uses BSD netcat
>>
>> Please fix the script name.
>>
>>> +# to make SOCKS5 or HTTPS proxy connections. It uses ALL_PROXY to determine the
>>> +# proxy server, protocol, and port. It uses NO_PROXY to skip using the proxy for
>>> +# a comma delimited list of hosts, host globs (*.example.com), IPs, or CIDR masks
>>> +# (192.168.1.0/24). It is known to work with both bash and dash shells.
>>> +#
>>> +# BSD netcat is provided by netcat-openbsd on Ubuntu and nc on Fedora.
>>> +#
>>> +# Example ALL_PROXY values:
>>> +# ALL_PROXY=socks://socks.example.com:1080
>>> +# ALL_PROXY=https://proxy.example.com:8080
>>> +#
>>> +# Copyright (c) 2013, Intel Corporation.
>>> +# All rights reserved.
>>
>> And please add a clear license here.
>
>
> Both fixed in the contrib branch. GPL license blurb added.
>
> Have you happened to have tested the script?
>
> Thanks Otavio.

No; not yet.

I am wondering if the code wouldn't be easier to read if done in
Python ... the ipv4 stuff is UGLY and hard to follow ... I understand
it is very simple script but maybe Python would be the right tool for
it. What do you think?

--
Otavio Salvador                             O.S. Systems
E-mail: otavio@ossystems.com.br  http://www.ossystems.com.br
Mobile: +55 53 9981-7854              http://projetos.ossystems.com.br
Darren Hart - Feb. 6, 2013, 12:16 a.m.
On 02/05/2013 04:09 PM, Otavio Salvador wrote:
> On Tue, Feb 5, 2013 at 10:04 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>> On 02/05/2013 03:51 PM, Otavio Salvador wrote:
>>> On Tue, Feb 5, 2013 at 8:52 PM, Darren Hart <dvhart@linux.intel.com> wrote:
>>>> oe-git-proxy.sh is a simple tool to be used via GIT_PROXY_COMMAND. It
>>>> uses BSD netcat to make SOCKS5 or HTTPS proxy connections. It uses
>>>> ALL_PROXY to determine the proxy server, protocol, and port. It uses
>>>> NO_PROXY to skip using the proxy for a comma delimited list of hosts,
>>>> host globs (*.example.com), IPs, or CIDR masks (192.168.1.0/24). It is
>>>> known to work with both bash and dash shells.
>>>>
>>>> V2: Implement recommendations by Enrico Scholz:
>>>>     o Use exec for the nc calls
>>>>     o Use "$@" instead of $* to avoid quoting issues inherent with $*
>>>>     o Use bash explicitly and simplify some of the string manipulations
>>>>     Also:
>>>>     o Drop the .sh in the name per Otavio Salvador
>>>>     o Remove a stray debug statement
>>>>
>>>> Signed-off-by: Darren Hart <dvhart@linux.intel.com>
>>>> Cc: Enrico Scholz <enrico.scholz@sigma-chemnitz.de>
>>>> Cc: Otavio Salvador <otavio@ossystems.com.br>
>>>> ---
>>>>  scripts/oe-git-proxy |  124 ++++++++++++++++++++++++++++++++++++++++++++++++++
>>>>  1 files changed, 124 insertions(+), 0 deletions(-)
>>>>  create mode 100755 scripts/oe-git-proxy
>>>>
>>>> diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy
>>>> new file mode 100755
>>>> index 0000000..4f1871a
>>>> --- /dev/null
>>>> +++ b/scripts/oe-git-proxy
>>>> @@ -0,0 +1,124 @@
>>>> +#!/bin/bash
>>>> +
>>>> +# oe-git-proxy.sh is a simple tool to be via GIT_PROXY_COMMAND. It uses BSD netcat
>>>
>>> Please fix the script name.
>>>
>>>> +# to make SOCKS5 or HTTPS proxy connections. It uses ALL_PROXY to determine the
>>>> +# proxy server, protocol, and port. It uses NO_PROXY to skip using the proxy for
>>>> +# a comma delimited list of hosts, host globs (*.example.com), IPs, or CIDR masks
>>>> +# (192.168.1.0/24). It is known to work with both bash and dash shells.
>>>> +#
>>>> +# BSD netcat is provided by netcat-openbsd on Ubuntu and nc on Fedora.
>>>> +#
>>>> +# Example ALL_PROXY values:
>>>> +# ALL_PROXY=socks://socks.example.com:1080
>>>> +# ALL_PROXY=https://proxy.example.com:8080
>>>> +#
>>>> +# Copyright (c) 2013, Intel Corporation.
>>>> +# All rights reserved.
>>>
>>> And please add a clear license here.
>>
>>
>> Both fixed in the contrib branch. GPL license blurb added.
>>
>> Have you happened to have tested the script?
>>
>> Thanks Otavio.
>
> No; not yet.
>
> I am wondering if the code wouldn't be easier to read if done in
> Python ... the ipv4 stuff is UGLY and hard to follow ... I understand
> it is very simple script but maybe Python would be the right tool for
> it. What do you think?

I had originally written the IP matching stuff in C, but converted to
shell to
keep this script self contained and easily deployable. The proper way to
do this
with python would require the use of external modules, which may or may
not be
installed on the target system. Although I suppose even just replacing this
script and all the math it does in python might be an improvement.
However, For
the purposes of a one-stop-shop example git proxy script for oe-core, I
think
bash still serves us best here. Otherwise, yes, I completely agree it's
horrible
to look at :-)

Patch

diff --git a/scripts/oe-git-proxy b/scripts/oe-git-proxy
new file mode 100755
index 0000000..4f1871a
--- /dev/null
+++ b/scripts/oe-git-proxy
@@ -0,0 +1,124 @@ 
+#!/bin/bash
+
+# oe-git-proxy.sh is a simple tool to be via GIT_PROXY_COMMAND. It uses BSD netcat
+# to make SOCKS5 or HTTPS proxy connections. It uses ALL_PROXY to determine the
+# proxy server, protocol, and port. It uses NO_PROXY to skip using the proxy for
+# a comma delimited list of hosts, host globs (*.example.com), IPs, or CIDR masks
+# (192.168.1.0/24). It is known to work with both bash and dash shells.
+#
+# BSD netcat is provided by netcat-openbsd on Ubuntu and nc on Fedora.
+#
+# Example ALL_PROXY values:
+# ALL_PROXY=socks://socks.example.com:1080
+# ALL_PROXY=https://proxy.example.com:8080
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# AUTHORS
+# Darren Hart <dvhart@linux.intel.com>
+
+# Locate the netcat binary
+NC=$(which nc 2>/dev/null)
+if [ $? -ne 0 ]; then
+	echo "ERROR: nc binary not in PATH"
+	exit 1
+fi
+METHOD=""
+
+# Test for a valid IPV4 quad with optional bitmask
+valid_ipv4() {
+	echo $1 | egrep -q "^([1-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])(\.([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])){3}(/(3[0-2]|[1-2]?[0-9]))?$"
+	return $?
+}
+
+# Convert an IPV4 address into a 32bit integer
+ipv4_val() {
+	IP="$1"
+	SHIFT=24
+	VAL=0
+	for B in ${IP//./ }; do
+		VAL=$(($VAL+$(($B<<$SHIFT))))
+		SHIFT=$(($SHIFT-8))
+	done
+	echo "$VAL"
+}
+
+# Determine if two IPs are equivalent, or if the CIDR contains the IP
+match_ipv4() {
+	CIDR=$1
+	IP=$2
+
+	if [ -z "${IP%%$CIDR}" ]; then
+		return 0
+	fi
+
+	# Determine the mask bitlength
+	BITS=${CIDR##*/}
+	if [ -z "$BITS" ]; then
+		return 1
+	fi
+
+	IPVAL=$(ipv4_val $IP)
+	IP2VAL=$(ipv4_val ${CIDR%%/*})
+
+	# OR in the unmasked bits
+	for i in $(seq 0 $((32-$BITS))); do
+		IP2VAL=$(($IP2VAL|$((1<<$i))))
+		IPVAL=$(($IPVAL|$((1<<$i))))
+	done
+
+	if [ $IPVAL -eq $IP2VAL ]; then
+		return 0
+	fi
+	return 1
+}
+
+# Test to see if GLOB matches HOST
+match_host() {
+	HOST=$1
+	GLOB=$2
+
+	if [ -z "${HOST%%$GLOB}" ]; then
+		return 0
+	fi
+
+	# Match by netmask
+	if valid_ipv4 $GLOB; then
+		HOST_IP=$(gethostip -d $HOST)
+		if valid_ipv4 $HOST_IP; then
+			match_ipv4 $GLOB $HOST_IP
+			if [ $? -eq 0 ]; then
+				return 0
+			fi
+		fi
+	fi
+
+	return 1
+}
+
+# If no proxy is set, just connect directly
+if [ -z "$ALL_PROXY" ]; then
+	exec $NC -X connect "$@"
+fi
+
+# Connect directly to hosts in NO_PROXY
+for H in ${NO_PROXY//,/ }; do
+	if match_host $1 $H; then
+		METHOD="-X connect"
+		break
+	fi
+done
+
+if [ -z "$METHOD" ]; then
+	# strip the protocol and the trailing slash
+	PROTO=$(echo $ALL_PROXY | sed -e 's/\([^:]*\):\/\/.*/\1/')
+	PROXY=$(echo $ALL_PROXY | sed -e 's/.*:\/\/\([^:]*:[0-9]*\).*/\1/')
+	if [ "$PROTO" = "socks" ]; then
+		METHOD="-X 5 -x $PROXY"
+	elif [ "$PROTO" = "https" ]; then
+		METHOD="-X connect -x $PROXY"
+	fi
+fi
+
+exec $NC $METHOD "$@"