[root@python ~]# cat testcount.txt

a 1.1.1.1

b 2.2.2.2

c 3.3.3.3

a 1.1.1.1

d 4.4.4.4

e 5.5.5.5

f 6.6.6.6

1.去重：

awk '!a[$1]++ {print}' testcount.txt 

a 1.1.1.1

b 2.2.2.2

c 3.3.3.3

d 4.4.4.4

e 5.5.5.5

f 6.6.6.6

2.统计第二列

awk '{a[$1]++} END{for (i in a) print a[i],i}' testcount.txt

2 a

1 b

1 c

1 d

1 e

1 f

3.统计日志中访问大于100次的IP

[root@python ~]# awk '{a[$1]++}END{for (i in a) {if (a[i] >1) print a[i],i}}' testcount.txt

2 a

过滤邮箱地址

[root@python ~]# egrep '[a-z0-9]{3,}\@[1-9a-z]{4,}\.[1-9a-z]{2,}' oldboy.txt

linuxdxw@onecloud.cn

[root@python ~]# egrep '[a-z0-9]{3,}\@[1-9a-z]{4,}\.[1-9a-z]{2,}' oldboy.txt

过滤ip地址

[root@python ~]# egrep '[0-9]{1,3}(\.[0-9]){3}' oldboy.txt

inet addr:10.0.0.8 Bcast:10.0.0.255 Mask:255.255.255.0

inet addr:10.0.0.8 Bcast:10.0.0.255 Mask:255.255.255.0

过滤mac地址

[root@python ~]# egrep '[0-9a-z]{2}(\:[0-9a-z]{2}){5}' oldboy.txt

link/ether 02:00:47:76:00:6c brd ff:ff:ff:ff:ff:ff