当前位置： 代码迷 >> Web前端 >> webgame 日记脚本-最终版
  详细解决方案

webgame 日记脚本-最终版

热度：761   发布时间：2012-07-20 10:38:30.0
webgame 日志脚本-最终版
0、testlogfor.sh
#!/bin/sh
echo `date`

export PATH="/usr/kerberos/bin:$PATH"
export KRB5CCNAME=/tmp/krb5cc_pub_$$
trap kdestroy 0 1 2 3 5 15
kinit -k -t /etc/krb5.keytab

k=0
while((k++<1))
do
  dd=`date +%Y-%m-%d -d " $k days ago"`
  echo $dd

  ssh root@10.3.17.26 "cat /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd}-* > /data/remote-log-server/log/promition/${dd}/promition.success.log.${dd};"

  scp root@10.3.17.26:/data/remote-log-server/log/promition/$dd/promition.success.log.${dd} /data/remote-log-server/log/gamelogin/wqo/test/promition.success.log.${dd}

  ssh root@10.3.17.26 "rm -f /data/remote-log-server/log/promition/$dd/promition.success.log.${dd};"

  echo "************ $dd promition.success end**************"

  scp root@10.3.17.26:/data/remote-log-server/log/gamelogin/${dd}/gamelogin.log /data/remote-log-server/log/gamelogin/wqo/test/gamelogin.log.${dd}

  echo "************ $dd gamelogin end**************"
done


1、test.sh
#!/bin/sh

cd /data/remote-log-server/log/gamelogin/wqo/test

if [ -f "flux-rate.log" ] ; then

   echo "*****flux-rate.log is existed**********"
   
   > flux-rate.log

fi

echo "*********start********"

m=0
while((m++<1))
do

##dd=`date +%Y-%m-%d -d " $m days ago " `
dd="2011-08-24"

if [ ! -f "gamelogin.log.$dd" ] ; then
    
   echo "*****gamelogin.log.$dd not existed*******"
   
   exit
fi
 

if [ ! -f "glogin.log.$dd" ] ; then    

   echo "***********glogin.log.$dd not exist, start to create it**************"

   awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd

   echo "***********glogin.log.$dd end***********"
 
else
   
   echo "************glogin.log.$dd is existed********************"

fi

if [ ! -f "glogin-uid.$dd" ] ; then
   echo "********************glogin-uid.$dd not exist, create id***********"
   awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd

else
   echo "********************glogin-uid.$dd is existed******"
fi


################################################################

echo "****for promition*****"

k=0
while((k++<1))
do

((n=k+m))

##tt=`date +%Y-%m-%d -d " $n  days ago " `
tt="2011-08-10"

  if [ -f "promition.success.log.$tt" ] ; then


      if [ ! -f "prom-suc.log.$tt" ] ; then

        echo "************prom-suc.log.$tt not exist , start to create it***********************"
  
        awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt

        echo "***********prom-suc.log.$tt end***************"

      else 
  
        echo "***********prom-suc.log.$tt is existed**********************"

      fi
       
      if [ ! -f "prom-suc-uid.$tt" ] ; then
         
        echo "***************prom-suc-uid.$tt is not exited, create**********************"
        
        awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt

      else
     
        echo "****************prom-suc-uid.$tt is existed********************"

      fi

      cat glogin-uid.$dd | wc -l  > tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log
      
      cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log

      echo $dd >>tmp-glog-prom-uid.log

      echo $tt >>tmp-glog-prom-uid.log

      cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log

      awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log

      echo "***********flux-rate.log end************"
      rm -f tmp*

  else

      echo "********promition.success.log.$tt not exit!***************"

  fi


done

done

##cat flux-rate.log | sort > webgame-total.csv

echo "*****************total end**********************"

echo "****************category start*****************************"

##sh testcatfor.sh

echo "****************category end ******************************"

echo  "***************service start ******************************"

##sh testservicefor.sh

echo  "***************service end *****************************"

echo "****************start insert data into db*****************************"
##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv

##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv

##/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csv



echo "****************data end*********************"



2、testfor.sh
#!/bin/sh

start=`date`
echo $start

cd /data/remote-log-server/log/gamelogin/wqo/test

if [ -f "flux-rate.log" ] ; then

   echo "*****flux-rate.log is existed**********"
   
   > flux-rate.log

fi

echo "*********start********"

m=0
while((m++<1))
do

dd=`date +%Y-%m-%d -d " $m days ago " `

if [ ! -f "gamelogin.log.$dd" ] ; then
    
   echo "*****gamelogin.log.$dd not existed*******"
   
   exit
fi
 

if [ ! -f "glogin.log.$dd" ] ; then    

   echo "***********glogin.log.$dd not exist, start to create it**************"

   awk 'BEGIN {FS="|"}; {if($2!=""){print $2 "|" $3  "|" $4}}' gamelogin.log.$dd | awk 'BEGIN {FS="|"}; { if(index($1,":")>0) {print substr($1, 1, index($1,":")-1) "|" $2 "|" $3 } else {print $1 "|" $2 "|" $3} }' | awk 'BEGIN {FS="|"}; {print  "|" $1 "|" $2 "|" $3 }' | sort| uniq -c |awk 'BEGIN {FS="|"}; {print gsub(/[[:blank:]]*/,"",$1) "|" $2 "|" $3 "|" $4}'| sort > glogin.log.$dd

   echo "***********glogin.log.$dd end***********"
 
else
   
   echo "************glogin.log.$dd is existed********************"

fi

if [ ! -f "glogin-uid.$dd" ] ; then
   echo "********************glogin-uid.$dd not exist, create id***********"
   awk 'BEGIN {FS="|"}; {print $2}' glogin.log.$dd | sort | uniq > glogin-uid.$dd

else
   echo "********************glogin-uid.$dd is existed******"
fi


################################################################

echo "****for promition*****"

k=0
while((k++<30))
do

((n=k+m))

tt=`date +%Y-%m-%d -d " $n  days ago " `

  if [ -f "promition.success.log.$tt" ] ; then


      if [ ! -f "prom-suc.log.$tt" ] ; then

        echo "************prom-suc.log.$tt not exist , start to create it***********************"
  
        awk 'BEGIN {FS=","}; {print  $6  " " $7 " " $5 }' promition.success.log.$tt | sort |uniq -c |awk 'BEGIN {FS=" "}; {print $1 "|" $4 "|" $2 "|" $3 }'| sort > prom-suc.log.$tt

        echo "***********prom-suc.log.$tt end***************"

      else 
  
        echo "***********prom-suc.log.$tt is existed**********************"

      fi
       
      if [ ! -f "prom-suc-uid.$tt" ] ; then
         
        echo "***************prom-suc-uid.$tt is not exited, create**********************"
        
        awk 'BEGIN {FS="|"}; {print $2}' prom-suc.log.$tt | sort | uniq > prom-suc-uid.$tt

      else
     
        echo "****************prom-suc-uid.$tt is existed********************"

      fi

      cat glogin-uid.$dd | wc -l  > tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt | wc -l >> tmp-glog-prom-uid.log

      cat prom-suc-uid.$tt glogin-uid.$dd | sort | uniq -d > tmp-match-uid.log
      
      cat tmp-match-uid.log | wc -l >> tmp-glog-prom-uid.log

      echo $dd >>tmp-glog-prom-uid.log

      echo $tt >>tmp-glog-prom-uid.log

      cat tmp-glog-prom-uid.log | xargs echo > tmp-flux-rate.log

      awk 'BEGIN {FS=" "}; {print $5 "," $4 "," $3/$2 "," $3 "," $2 "," $1 }' tmp-flux-rate.log >> flux-rate.log

      echo "***********flux-rate.log end************"
      rm -f tmp*

  else

      echo "********promition.success.log.$tt not exit!***************"

  fi


done

done

cat flux-rate.log | sort > webgame-total.csv

echo "*****************total end**********************"

echo "****************category start*****************************"

sh testcatfor.sh

echo "****************category end ******************************"

echo  "***************service start ******************************"

sh testservicefor.sh

echo  "***************service end *****************************"

echo "****************start insert data into db*****************************"
/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_total webgame-total.csv

/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_category webgame-cat.csv

/usr/bin/python python_webgame.py webgametest "gametestdb" webgame_test webgame_service webgame-cat-service.csv



echo "****************data end*********************"

end=`date`
echo $end


3、testcatfor.sh
#!/bin/sh


if [ -f "flux-cat-rate.log" ]; then

    echo "****flux-cat-rate.log is existed******"
    
    > flux-cat-rate.log
fi


echo "**********analysis category and service number**************"

n=0
while((n++<1))
do

  tt=`date +%Y-%m-%d -d " $n days ago" `

  if [ ! -f "glogin.log.$tt" ] ; then
     echo "********glogin.log.$tt not existed********************"
  fi

  echo "****glogin.log.$tt is existed********"


  m=0
  while((m++<=30))
  do
   
  ((p=m+n))

  dd=`date +%Y-%m-%d -d " $p days ago " `

  if [ ! -f "prom-suc.log.$dd" ] ; then

      echo "********prom-suc.log.$dd not existed****************"

  else

    echo "***********prom-suc.log.$dd is existed*******************"

    awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log

    awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log

    catnum=`cat tmp-pcat-name.log | wc -l`

    echo "**$catnum**"

    #####for######
    k=0
    while((k++<${catnum}))
    do

      catcontent=`sed -n "${k} p" tmp-pcat-name.log`

      echo "---------$catcontent------------"

      cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log

      awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-uid.log


      ####tt=`date +%Y-%m-%d -d " 1 days ago" `

      cat glogin.log.$tt | grep $catcontent > tmp-glogin-cat.log

      awk 'BEGIN {FS="|"}; {print $2 "|" $3}' tmp-glogin-cat.log | sort | uniq > tmp-glogin-cat-uid.log

      cat tmp-prom-suc-cat-uid.log | wc -l > tmp-cat-prom-glogin-uid.log

      cat tmp-glogin-cat-uid.log | wc -l >> tmp-cat-prom-glogin-uid.log

      cat tmp-prom-suc-cat-uid.log tmp-glogin-cat-uid.log | sort | uniq -d | wc -l >> tmp-cat-prom-glogin-uid.log

      echo $catcontent >> tmp-cat-prom-glogin-uid.log
 
      echo $dd >> tmp-cat-prom-glogin-uid.log

      echo $tt >> tmp-cat-prom-glogin-uid.log

      cat tmp-cat-prom-glogin-uid.log | xargs echo > tmp-cat-rate.log

      awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-cat-rate.log >> flux-cat-rate.log

      done

  
      echo "**********flux-cat-rate.log***************"

    fi

  done

done


cat flux-cat-rate.log | sort > webgame-cat.csv


3、testservicefor.sh
#!/bin/sh


if [ -f "flux-service-rate.log" ]; then

    echo "****flux-service-rate.log is existed******"
    
    > flux-service-rate.log
fi


echo "**********analysis category and service number**************"

n=0
while((n++<1))
do

  tt=`date +%Y-%m-%d -d " $n days ago" `

  if [ ! -f "glogin.log.$tt" ] ; then
     echo "********glogin.log.$tt not existed********************"
  fi

  echo "****glogin.log.$tt is existed********"


  m=0
  while((m++<30))
  do
   
  ((p=m+n))

  dd=`date +%Y-%m-%d -d " $p days ago " `

  if [ ! -f "prom-suc.log.$dd" ] ; then

      echo "********prom-suc.log.$dd not existed****************"

  else

    echo "***********prom-suc.log.$dd is existed*******************"

    awk 'BEGIN {FS="|"}; {print $3 "|" $4} ' prom-suc.log.$dd | sort | uniq > tmp-pcat.log

    awk 'BEGIN {FS="|"}; {print $1 } ' tmp-pcat.log | sort | uniq > tmp-pcat-name.log

    catnum=`cat tmp-pcat-name.log | wc -l`

    echo "**$catnum**"

    #####for######
    k=0
    while((k++<${catnum}))
    do

      catcontent=`sed -n "${k} p" tmp-pcat-name.log`

      echo "---------$catcontent------------"

      cat prom-suc.log.$dd | grep $catcontent > tmp-prom-suc-cat.log

      
    
      awk 'BEGIN {FS="|"}; {print $2 "|" $3  "|" $4}' tmp-prom-suc-cat.log | sort | uniq > tmp-prom-suc-cat-service-uid.log

      awk 'BEGIN {FS="|"}; {print $3}' tmp-prom-suc-cat-service-uid.log | sort | uniq > tmp-pcat-service-name.log

      servicenum=`cat tmp-pcat-service-name.log | wc -l`

      echo  "+++++$servicenum++"


      ######for  service##########

      g=0
      while((g++<${servicenum}))
      do
    
        servicecontent=`sed -n "${g} p" tmp-pcat-service-name.log`

        echo "---------------$servicecontent------------"
        
        cat glogin.log.$tt | grep $catcontent > tmp-glogin-catser.log

     ######   echo "*****11111********" 
          
        cat tmp-glogin-catser.log | grep $servicecontent > tmp-glogin-cat-service.log

      #####  echo "*****22222*********"
        awk 'BEGIN {FS="|"}; {print $2 "|" $3 "|" $4 }' tmp-glogin-cat-service.log | sort | uniq > tmp-glogin-cat-service-uid.log

        cat tmp-prom-suc-cat-service-uid.log | grep $servicecontent | wc -l > tmp-service-prom-glogin-uid.log

        cat tmp-glogin-cat-service-uid.log | wc -l >> tmp-service-prom-glogin-uid.log

        cat tmp-prom-suc-cat-service-uid.log tmp-glogin-cat-service-uid.log | sort | uniq -d | wc -l >> tmp-service-prom-glogin-uid.log

        echo $catcontent >> tmp-service-prom-glogin-uid.log

        echo $servicecontent >> tmp-service-prom-glogin-uid.log
 
        echo $dd >> tmp-service-prom-glogin-uid.log

        echo $tt >> tmp-service-prom-glogin-uid.log

        cat tmp-service-prom-glogin-uid.log | xargs echo > tmp-service-rate.log

        awk 'BEGIN {FS=" "}; {print $4 "," $5 "," $6 "," $7 "," $3/$1 "," $3 "," $1 "," $2 }' tmp-service-rate.log >> flux-service-rate.log

      done
    
     done

  
     echo "**********flux-service-rate.log***************"

    fi

  done

done


cat flux-service-rate.log | sort > webgame-cat-service.csv


4、python_webgame.py
已可从第一行读csv
#!/usr/bin/python
#python_webgame.py
import sys, csv
import MySQLdb

def main(user, pwd, db, table, csvfile):

    try:
        conn = getconn(user, pwd, db)
    except MySQLdb.Error, e:
        print "Error %d: %s" % (e.args[0], e.args[1])
        sys.exit(1)

    cursor = conn.cursor()

    loadcsv(cursor, table, csvfile)

    cursor.close()
    conn.close()

def getconn(user, pwd, db):
    conn = MySQLdb.connect(host = "10.3.18.158",
                           user = user,
                           passwd = pwd,
                           db = db)
    return conn

def nullify(L):
    """Convert empty strings in the given list to None."""

    #helper function
    def f(x):
        if(x == ""):
            return None
        else:
            return x

    return [ f(x) for x in L ]

def loadcsv(cursor, table, filename):

    """
    Open a csc file and load it into a sql table.
    Assumptions:
     - the first line in the file is a header
    """

    f = open(filename)

    c = csv.reader(f)

    header = c.next()

    numfields = len(header)

    query = buildInsertCmd(table, numfields)
    
    f.seek(0)

    for line in c:
        if len(line)<1: continue
        vals = nullify(line)
        cursor.execute(query, vals)

    return


def buildInsertCmd(table, numfields):

    """
    example:
    >>>buildInserCmd("foo", 3)
    'insert int foo values (%s,%s,%s)'
    """

    assert(numfields > 0)
    placeholders = (numfields-1) * "%s, " + "%s"
    query = ("insert into %s" % table) + (" values (null, %s)" % placeholders)
    return query

if __name__ == '__main__':

    args = sys.argv[1:]
    if(len(args) < 5):
        print "error: arguments: user \"password\" db table csvfile"
        sys.exit(1)

    main(*args)
1 楼 printfabcd 2011-11-05  
export PATH="/usr/kerberos/bin:$PATH" 
export KRB5CCNAME=/tmp/krb5cc_pub_$$ 
trap kdestroy 0 1 2 3 5 15 
kinit -k -t /etc/krb5.keytab 
请问这几行代码是什么意思？
我一直再用，可是不懂是做什么的
  相关解决方案

代码迷 - 您身边的软件异常，代码异常，编程异常助手（发现，解决，分享）
Copyright © 2009-2013 DAIMAMI.COM. All rights reserved.