<?xml version="1.0" encoding="UTF-8"?>
<site
base_url="http://site1.ru/"
store_into="/path/site1.ru/Sitemap.xml"
verbose="1"
>
<urllist path="/tmp/webcheck/url_list.txt" encoding="UTF-8" />
</site>

<?xml version="1.0" encoding="UTF-8"?>
<site
base_url="http://site1.ru/"
store_into="/path/site1.ru/Sitemap.xml.gz"
verbose="1"
>
<urllist path="/tmp/webcheck/url_list.txt" encoding="UTF-8" />
</site>

<?xml version="1.0" encoding="UTF-8"?>
<site
base_url="http://site1.com/"
store_into="/path/site1.com/Sitemap.xml"
verbose="1"
>
<urllist path="/tmp/webcheck/url_list.txt" encoding="UTF-8" />
</site>

#!/bin/sh
lastmod=`date +"%Y-%m-%d"`
folder=/tmp/webcheck
if [ ! -d $folder ]
then
mkdir $folder
fi
cd $folder
/usr/local/bin/webcheck.py -afq http://site1.ru/
cat urllist.html \
| awk '{if (index($3,"internal")>0 && index($2,"?")==0) {print substr($2,7,length($2)-7)}
| awk '!/(\/|\.css|\.gif|\.jpeg|\.jpg|\.js)$/' | sort \
| awk '{if (index($1,"catalog")>0) {print $1 " changefreq=daily priority=1.0 lastmod='$la
else {print $1 " changefreq=weekly priority=0.5 lastmod='$lastmod'"}}' > url_list.txt
/usr/local/bin/python /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=/etc/sitemap/sitemapgen.site1.ru.xml.conf
/usr/local/bin/python /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=/etc/sitemap/sitemapgen.site1.ru.xml.gz.conf
rm -Rf $folder

#!/bin/sh
lastmod=`date +"%Y-%m-%d"`
folder=/tmp/webcheck
if [ ! -d $folder ]
then
mkdir $folder
fi
cd $folder
/usr/local/bin/webcheck.py -afq http://site1.com/
cat urllist.html \
| awk '{if (index($3,"internal")>0 && index($2,"?")==0) {print substr($2,7,length($2)-7)}
| awk '!/(\/|\.css|\.gif|\.jpeg|\.jpg|\.js)$/' | sort \
| awk '{if (index($1,"catalog")>0) {print $1 " changefreq=daily priority=1.0 lastmod='$la
else {print $1 " changefreq=weekly priority=0.5 lastmod='$lastmod'"}}' > url_list.txt
/usr/local/bin/python /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=/etc/sitemap/sitemapgen.site1.com.conf
rm -Rf $folder

...
The Sitemap type is WEB Sitemap.
Opened URLLIST file: /tmp/webcheck/url_list.txt
Sorting and normalizing collected URLs.
Writing Sitemap file "/path/site1.ru/Sitemap.xml" with 54 URLs
Notifying search engines.
Notifying: www.google.com
Count of file extensions on URLs:
54 .html
Number of errors: 0
Number of warnings: 0
Reading configuration file: /sitemap/sitemapgen.site1.ru.xml.gz.conf
The Sitemap type is WEB Sitemap.
Opened URLLIST file: /tmp/webcheck/url_list.txt
Sorting and normalizing collected URLs.
Writing Sitemap file "/path/site1.ru/Sitemap.xml.gz" with 54 URLs
Notifying search engines.
Notifying: www.google.com
Count of file extensions on URLs:
54 .html
Number of errors: 0
Number of warnings: 0

# /bin/sh /<path>/sitemap/anteh.ru.sitemapgen
env: python: No such file or directory
cat: urllist.html: No such file or directory
/<path>/sitemap/anteh.ru.sitemapgen: /usr/local/bin/python: not found
/<path>/sitemap/anteh.ru.sitemapgen: /usr/local/bin/python: not found

#!/bin/sh
lastmod=`date +"%Y-%m-%d"`
folder=/tmp/webcheck
if [ ! -d $folder ]
then
mkdir $folder
fi
cd $folder
/usr/local/bin/webcheck.py -afq http://site.ru/
cat urllist.html \
| awk '{if (index($3,"internal")>0 && index($2,"?")==0) {print substr($2,7,length($2)-7)}}' \
| awk '!/(\/|\.css|\.gif|\.jpeg|\.png|\.ogg|\.ogv|\.dll|\.rar|\.gz|\.pdf|\.tar|\.jpg|\.js|\.mpeg)$/' | sort \
| awk '{if (index($1,"catalog")>0) {print $1 " changefreq=daily priority=1.0 lastmod='$lastmod'"} \
else {print $1 " changefreq=weekly priority=0.5 lastmod='$lastmod'"}}' > url_list.txt
/usr/local/bin/python2 /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=/etc/sitemap/sitemapgen.anteh.ru.xml.conf
/usr/local/bin/python2 /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=/etc/sitemap/sitemapgen.anteh.ru.xml.gz.conf
rm -Rf $folder

...
Reading configuration file: /<path>/sitemapgen.site.ru.xml.conf
The Sitemap type is WEB Sitemap.
Opened URLLIST file: /tmp/webcheck/url_list.txt
Sorting and normalizing collected URLs.
Writing Sitemap file "/<path>/site.ru/Sitemap.xml" with 74 URLs
Notifying search engines.
Notifying: www.google.com
Count of file extensions on URLs:
73 .html
Number of errors: 0
Number of warnings: 0
Reading configuration file: /<path>/sitemapgen.site.ru.xml.gz.conf
The Sitemap type is WEB Sitemap.
Opened URLLIST file: /tmp/webcheck/url_list.txt
Sorting and normalizing collected URLs.
Writing Sitemap file "/<path>/site.ru/Sitemap.xml.gz" with 74 URLs
Notifying search engines.
Notifying: www.google.com
Count of file extensions on URLs:
73 .html
Number of errors: 0
Number of warnings: 0
#

#!/usr/local/bin/bash
set -o nounset
set -o errexit
#/usr/local/bin/bash /<script directory>/sitemap/sitemapgen

#=== CONFIG ============================
#Apache virtual host configuration file
VHostNameOr="/usr/local/etc/apache24/extra/httpd-vhosts.conf"
#Add host to handle
Host=(anteh.ru anteh.com adc.anteh.ru site.ru)
#Soft apache restart. Handle each host separately
MODE="seq"
#Hard apache restart. Handle all hosts simultaneously. apachectl restart 
#MODE="all"
#Dir with config for sitemap files
HostConfDir="/<script directory>/sitemap"
#---------------------------------------



host_handle()
{
lastmod=`date +"%Y-%m-%d"`
folder=/tmp/webcheck
if [ ! -d $folder ]
then
mkdir $folder
fi
cd $folder
/usr/local/bin/webcheck.py -afq http://$1/
cat urllist.html \
| awk '{if (index($3,"internal")>0 && index($2,"?")==0) {print substr($2,7,length($2)-7)}}' \
| awk '!/(\/|\.css|\.gif|\.jpeg|\.png|\.ogg|\.ogv|\.dll|\.rar|\.gz|\.pdf|\.tar|\.jpg|\.js|\.mpeg)$/' | sort \
| awk '{if (index($1,"catalog")>0) {print $1 " changefreq=daily priority=1.0 lastmod='$lastmod'"} \
else {print $1 " changefreq=weekly priority=0.5 lastmod='$lastmod'"}}' > url_list.txt
/usr/local/bin/python2 /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=$2/sitemapgen.$1.xml.conf
/usr/local/bin/python2 /usr/local/lib/python2.7/site-packages/sitemap_gen.py \
--config=$2/sitemapgen.$1.xml.gz.conf
rm -Rf $folder
}


#${Hosts[@]}  $VHostName
offf(){
#=== Correction of Hosts for regular expressions
declare -a HostsReg=(${Hosts[@]})  
if (( ${#HostsReg[@]}>0 )); then
    i=-1
    for a in "${HostsReg[@]}"; do
      i=$((i+1))
 HostsReg[i]=`echo $a | sed 's/\./\\\./g'`
    done
fi
echo "=- HOSTS: ${Hosts[@]} =="
readarray vh < $VHostNameOr

#Handle array
if (( ${#vh[@]}>0 )); then
    i=-1
    start=0
    last=0
    SecRulEng_present=false
    cat /dev/null > $VHostNameOr
    for a in "${vh[@]}"; do #Handle all file
     i=$((i+1))
 if [[ $a =~ ^[[:space:]]*\<VirtualHost.*\>{1}[[:space:]]*$ ]]; then
      start=$i
       fi
 if [[ $a =~ ^[[:space:]]*\</VirtualHost\>[[:space:]]*$ ]]; then
          flag=true
          str=$start
         SecRulEng_present=false
            while [ $str -le $i ] #Search ServerName marker
            do
                if $flag ;then #FIRST
               if [[ ${vh[$str]} =~ ^[[:space:]]*ServerName[[:space:]]*.*$ ]]; then
                   for b in "${HostsReg[@]}"; do #Find host
                             if [[ ${vh[$str]} =~ ^[[:space:]]*ServerName[[:space:]]*$b[[:space:]]*$ ]]; then
                               echo ${vh[$str]}
                           flag=false
                         str=$[$start-1]
                            break
                          fi
                     done
                   fi
             else
                   #Handle Vhost block SECOND
                 if [[ ${vh[$str]} =~ ^[[:space:]]*SecRuleEngine[[:space:]]*.*[[:space:]]*$ ]]; then
                    vh[$str]="SecRuleEngine Off"$'\n'
                        SecRulEng_present=true #SecRuleEngine was changed no need to create
                    fi
             fi
         str=$[$str+1]
          done

          while [ $last -lt $i ] #Add Block to out FIRST
     do
             #echo ${vh[$last]} >> "$VHostNameOr"
               printf "%s" "${vh[$last]}" >> "$VHostNameOr"
           if ! $flag && ! $SecRulEng_present ;then
               if [[ ${vh[$last]} =~ ^[[:space:]]*\<VirtualHost.*\>{1}[[:space:]]*$ ]]; then
                    #echo "SecRuleEngine Off" >> "$VHostNameOr"
                      printf "%s\n" "SecRuleEngine Off" >> "$VHostNameOr"
                    fi
                fi
          last=$[$last+1]
            done
       fi
    done
    while [ $last -lt ${#vh[@]} ] #Add Block to out SECOND
    do
        #echo ${vh[$last]} >> "$VHostNameOr"
 printf "%s" "${vh[$last]}" >> "$VHostNameOr"
   last=$[$last+1]
    done
fi
} 


declare -a Hosts=()
#=== Check mod_security LoadModule
mod_security_activated=false
if echo `apachectl -M` | egrep -q '.*security2_module \(shared\).*';then
    mod_security_activated=true
    echo "=- mod_security Detected =="
fi
if $mod_security_activated ;then #Change SecRuleEngine On to Off if mod_security loaded

    #=== Rezerv copy create ===============
    VHostName=$VHostNameOr"_rzq0"
    i=0
    while [ -f $VHostName ]
    do
  if [ $i -ge 5 ];then    #Create only 5 rezerved copy cyclic
            i=0
        rm $VHostNameOr"_rzq$i"
      fi
 VHostName=$VHostNameOr"_rzq$i"
   i=$[$i+1]
    done
    cp $VHostNameOr $VHostName #Reserv copy
    echo "=- Rezerv file: "$VHostName


    if [ $MODE == "all" ];then
  Hosts=("${Host[@]}")
     offf
       apachectl restart
  ia=0
       while [ $ia -lt ${#Hosts[@]} ] #Add Block to out SECOND
    do
     host_handle ${Hosts[$ia]} $HostConfDir
     ia=$[$ia+1]
    done
       cp $VHostName $VHostNameOr
 apachectl restart
    fi

    if [ $MODE == "seq" ];then
     ib=0
       while [ $ib -lt ${#Host[@]} ] #Add Block to out SECOND
     do
     Hosts=("${Host[$ib]}")
           offf
       apachectl -k graceful
      host_handle ${Hosts[0]} $HostConfDir
       cp $VHostName $VHostNameOr
         ib=$[$ib+1]
    done
       #cp $VHostName $VHostNameOr
        apachectl -k graceful
    fi
    echo "=- Return SecRuleEngine to last value =="
    exit 0
else
    echo "=- Mod_security Not Detected =="

    ib=0
    while [ $ib -lt ${#Host[@]} ] #Add Block to out SECOND
    do
        Hosts=("${Host[$ib]}")
        host_handle ${Hosts[0]} $HostConfDir
        ib=$[$ib+1]
    done
    apachectl -k graceful

fi
exit 0

 List of broken links and other issues

There are issues with the URLs listed below. The table summarizes the issues and suggested actions by HTTP response status code.
Code    Occurrences     What to do
400     21      This is usually the sign of a malformed URL that cannot be parsed by the server. Check the syntax of the link.

error Line: 34 https://anteh.ru/../apg2.html
    Status: 400 Bad Request

    This is usually the sign of a malformed URL that cannot be parsed by the server. Check the syntax of the link. 
error Line: 39 https://anteh.ru/../apg7.html
    Status: 400 Bad Request

    This is usually the sign of a malformed URL that cannot be parsed by the server. Check the syntax of the link.
...