Example 1

proxy="10.10.10.10:8080"

cnt=0
total=$(cat toto | grep bff | wc -l | awk '{print $1}')
for site in $(cat toto)
do
(( cnt = cnt + 1 ))
filename=$(echo $site | rev | cut -d'/' -f1 | rev)
echo "**** $cnt / $total : $filename $site"
curl --insecure --proxy $proxy $site -o $filename
done

Download all files from a folder:

# wget --no-parent -r http://WEBSITE.com/DIRECTORY

[root@gpfs01 TEMPO]# wget -r -l 3 --no-parent --reject-regex=repodata  https://public.dhe.ibm.com/aix/freeSoftware/aixtoolbox/RPMS/ppc-7.2/

Example1

#!/usr/bin/bash

proxy="10.10.10.10:8080"
export https_proxy="http://$proxy/"
export http_proxy="http://$proxy/"
export ftp_proxy="http://$proxy/"

if [ -z "$1" ]; then
        echo "Download patches from IBM with provided saved IBM Fix Central html page"
        echo "Syntax: $0 FILE"
        exit 1
fi
SOURCEFILE=$1


for BFF in $(cat $SOURCEFILE|grep ibm-download-link |awk '{print $3}'|cut -c6-|tr -d '"')
do
  if [ -f $BFF ]; then
    echo "Skipping file $BFF..."
  else
    echo "Downloading file $BFF:"
        wget -e use_proxy=on --no-check-certificate --debug --no-glob $BFF
  fi
done

Example2

#!/usr/bin/bash
# Download IBM rpm

proxy="10.10.10.10:8080"
export https_proxy="http://$proxy/"
export http_proxy="http://$proxy/"
export ftp_proxy="http://$proxy/"

TMPDIR=/tmp/rpm.$(date +%Y-%m-%d)
SOURCEFILE=alpha.html
wget -e use_proxy=on --no-check-certificate --debug --no-glob  https://www.ibm.com/developerworks/aix/library/aix-toolbox/alpha.html

mkdir $TMPDIR
cd $TMPDIR
for RPM in $(cat /suma/scripts/$SOURCEFILE | sed 's/href="/\ /g' | tr ' ' '\n' | grep rpm | grep RPMS | grep -v SRPMS | sed 's/"//g')
do
  directory=$(echo $RPM | awk -F'RPMS' '{print $2}' | awk -F'/'  '{print $2}')
  mkdir -p $directory
  cd $directory
  echo "Downloading file $RPM:"
  wget -e use_proxy=on --no-check-certificate --debug --no-glob $RPM
  cd $TMPDIR
done