#!/bin/sh

# todo parameters for website url or config file, copy dir, enabled git

dest_dir="/data"
sourcefile="urls"
git_enabled=true

wait_between=3600


# This script watch for website in conf file and copy them statically in git repositories
mkdir -p logs
mkdir -p clones
cd clones

while read -r line ; do
    if [ -z "$line" ] ; then continue ; fi
    mkdir -p "../logs/$line/"
    wget -c -w 10 --random-wait -x -m -p --follow-ftp -k "$line" 2> "../logs/$line/err.log" &
done < "../$sourcefile"