#!/bin/sh

# todo parameters for website url or config file, copy dir, enabled git

dest_dir="/data"
sourcefile="urls"
git_enabled=true

wait_between=3600


# This script watch for website in conf file and copy them statically in git repositories
mkdir -p logs
mkdir -p clones
cd clones

while read -r line ; do
    if [ -z "$line" ] ; then continue ; fi
    wget -c -w 10 --random-wait -x -r "$line" > "../logs/$line/out.log" 2> "../logs/$line/err.log" &
done < "../$sourcefile"