website_hoover/main.sh

21 lines
480 B
Bash
Raw Normal View History

2020-10-28 17:23:18 +00:00
#!/bin/sh
# todo parameters for website url or config file, copy dir, enabled git
dest_dir="/data"
sourcefile="urls"
git_enabled=true
wait_between=3600
# This script watch for website in conf file and copy them statically in git repositories
mkdir -p logs
mkdir -p clones
cd clones
while read -r line ; do
if [ -z "$line" ] ; then continue ; fi
wget -c -w 10 --random-wait -x -r "$line" > "../logs/$line/out.log" 2> "../logs/$line/err.log" &
done < "../$sourcefile"