Redis

Redis是一个开源的使用ANSI C语言编写、支持网络、可基于内存亦可持久化的日志型、Key-Value数据库,普遍用于提供高速存取服务,特别对于反复使用的数据,有着得天独厚的优势。

Redis运行时的数据保存在内存中,也可以定时将数据持久化到磁盘中或者通过日志输出。

Redis相比普通的键值对存储,具有一系列强力的命令支持不同的数据格式,比如字符串、哈希值、列表、集合和有序集合,甚至是位图或HyperLogLogs数据。

首先我们从redis官网https://redis.io/download下载redis数据库(Linux版),Windows版可从https://github.com/tporadowski/redis下载,但版本更新相对较慢。

1
2
3
4
5
6
7
8
$ wget https://download.redis.io/releases/redis-6.0.9.tar.gz
$ tar xzf redis-6.0.9.tar.gz
$ cd redis-6.0.9
$ make
# 初始化Redis服务, 默认监听6379端口
$ src/redis-server -port 6655
# Server initialized
# Ready to accept connections

进入cmd或者终端,输入redis-cli进入Redis命令行模式。

列举几个简单的系统终端命令:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
## 用SET命令保存一个键值对
SET name "Florance"
# OK

## 用EXIST命令检测某个键是否存在
EXISTS name
# (integer) 1

## 用EXPIRE设置一个键值对的过期秒数,或者使用EXPIREAT以UNIX时间戳的形式设置过期时间,一般用作缓存
GET name
# "Florance"
EXPIRE name 2
# (integer) 1
## 等待>2s
GET name
# (nil) ## null响应,即没有找到该键

用DEL命令可以删除键和值
SET total 1
DEL total
GET total
# (nil)

接下来使用python操纵redis:

1
2
3
# pip install redis
# 启动redis Windows
redis-cli -h 127.0.0.1 -p 6379

现列举几个危险的指令,

1
2
redis.flushdb() # delete database 
redis.flushall() # delete all databases

Connect

redis-py提供了StrictRedisRedis用于创建Redis对象,区别是前者只支持标准的Redis命令和语法,后者增加了一些扩展命令。

这里我们使用严格遵循标准Redis命令的StrictRedis模块:

1
2
3
4
5
6
7
8
9
from redis import StrictRedis
HOST = 'localhost'
PASS = ''
PORT = 6379
## 在Redis内部,数据库的编号是一个整数,共有0-16号数据库,默认客户端连接到的数据库是0号数据库,可以通过修改redis.conf更改默认数据库
DB = 0

redis = StrictRedis(host=HOST, password=PASS, port=PORT, db=DB)
redis2 = StrictRedis(host=HOST, password=PASS, port=PORT, db=DB +1)
1
2
3
4
5
6
7
from redis import StrictRedis, ConnectionPool
# TCP ://[:password]@host:port/db
# TCP+SSL://[:password]@host:port/db
# UNIX socket://[:password]@/path/to/socket.sock?db=db
url = 'redis://:@localhost:6379/0'
pool = ConnectionPool.from_url(url)
redis = StrictRedis(connection_pool=pool)

KEY

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
## 设置键值对
redis.set('Animal', 'Cat')
# True

## 获取键值对
redis.get('Animal')
# b'Cat'

## 批量设置键和值
redis.mset({'Name': 'Luis','name':'John','animal':'Dog','nothing': 'NA','age':'20'})
## nx: if not exists
redis.msetnx({'Name': 'Linda','nothing': ''})

print(redis.getset('Animal', "Rabbit"))
# b'Cat'
print(redis.get('Animal'))
# b'Rabbit'
print(redis.mget(['name', 'Name','age']))
# [b'John', b'Luis', b'20']
redis.setrange('name', 3, 'alex') # 3 offset
print(redis.get('name'))
# b'Johalex'
redis.setnx('newname', 'Lucy') # if key not exists
# True
redis.setex('it', 1, 'expires in 1s')
# True
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 如果键不存在,会自动创建该键(初始值为0)然后将值加1
redis.incr('age', 1)
redis.decr('age', 5) ## -5

redis.append('name', 'OK')
print(redis.mget(['age', 'name','it']))
# [b'16', b'JohalexOK', None]
print(redis.substr('Animal', 1, 4))
print(redis.getrange('Animal', 1, 4))
# b'abbi'
# b'abbi'
redis.exists('name')
# True
redis.type('name')
# b'string'
redis.keys('n*')
# [b'newname', b'name', b'nothing']
redis.randomkey()
# b'Animal'
redis.rename('name', 'nickname')
# True

## animal键值2s后过期
redis.expire('animal', 2)
1
2
3
4
5
6
7
8
9
10
11
# ....2s.....
redis.ttl('animal')
# -2
redis.delete('Animal')
print(redis.mget(['name','nickname', 'animal','Animal']))
# [None, b'JohalexOK', None, None]

redis.move('age', 1) # 2 database name
print(redis1.get('age'))
# b'20'

List

You can also practice list-like-python on redis.

1
2
3
4
5
6
7
8
9
10
11
12
## 向list末尾追加数据
redis.rpush('list', 1, 2, 3,4,5)
## 向list开头追加数据
redis.lpush('list', 0)
redis.rpush('list', 1, 2, 3,4,5)
redis.lpush('list', 0)
length = redis.llen('list')
print(length)
# 12
print(redis.lrange('list', 0, length))
# [b'0', b'0', b'1', b'2', b'3', b'4', b'5', b'1', b'2', b'3', b'4', b'5']
redis.ltrim('list', 1, 5)
1
2
3
4
5
def getList(name):
length = redis.llen(name)
return(redis.lrange(name, 0, length))
#== redis.zrange(name, 0, -1)
## redis.zrange(name, 0, -1, desc=True)
1
2
3
4
5
6
7
8
9
10
print(redis.lindex('list', 1))
# b'0'
## 删除队列的第一个元素
print(redis.lpop('list'))
# # b'0'
## 删除队列的最后一个元素
print(redis.rpop('list'))
# b'5'
getList("list")
# [b'0', b'1', b'2', b'3', b'4', b'5', b'1', b'2', b'3', b'4']
1
2
3
4
5
6
7
redis.lrem('list', 2, 3) # remove double 3
# [b'0', b'1', b'2', b'4', b'5', b'1', b'2', b'4']
redis.lset('list', 1, 5) ## 通过索引来设置元素的值
# [b'0', b'5', b'2', b'4', b'5', b'1', b'2', b'4']
redis.ltrim('list', 1, 5) ## 保留索引为1-5的值
getList("list")
# [b'5', b'2', b'4', b'5', b'1']
1
2
3
4
5
6
7
8
print(redis.rpoplpush('list', 'list2')) # move 1 to list2
# b'1'
print(redis.blpop('list'))
# (b'list', b'5')
print(redis.brpop('list'))
# (b'list', b'5')
getList("list")
# [b'2', b'4']

Set

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
print(redis.sadd('menu', 'dessert', 'Tea', 'Coffee',"Milk","Water","Berberage","Beer"))
print(redis.sadd('menu2', "Berberage","Beer"))
print(redis.srem('menu', 'Drink')) # return index if not exists return 0
print(redis.spop('menu')) # random pop
print(redis.smove('menu', 'menu2', 'Milk')) # delete Milk from menu move it to menu2
print(redis.scard('menu')) # number of elements in menu
print(redis.smembers('menu')) # show all elements in menu
print(redis.srandmember('menu')) # random element
print(redis.sismember('menu', 'Beer')) # if element Beer in set menu
print(redis.sinter(['menu', 'menu2'])) # intersect
print(redis.sinterstore('inttag', ['menu', 'menu2']))
print(redis.sunion(['menu', 'menu2']))
print(redis.sunionstore('inttag', ['menu', 'menu2']))
print(redis.sdiff(['menu', 'menu2']))
print(redis.sdiffstore('inttag', ['menu', 'menu2']))
# 7
# 2
# 0
# b'dessert'
# True
# 5
# {b'Coffee', b'Beer', b'Berberage', b'Water', b'Tea'}
# b'Coffee'
# True
# {b'Beer', b'Berberage'}
# 2
# {b'Coffee', b'Beer', b'Berberage', b'Milk', b'Water', b'Tea'}
# 6
# {b'Water', b'Coffee', b'Tea'}
# 3

Sorted Set

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
print(redis.zadd('animals', 5, 'Cat', 10, 'Dog',15,'Rabbit',45,'Chicken',100,'lynx',250,'bear'))
print(redis.zrem('animals', 'lynx')) # you delete 1
print(redis.zincrby('animals', 'Cat', -2)) # set 5-2 if Cat exists otherwise add '-2' Cat

print(redis.zrank('animals', 'Rabbit'))
print(redis.zrevrank('animals', 'Rabbit'))
print(redis.zrevrange('animals', 0, 3))
print(redis.zrangebyscore('animals', 50, 250))
print(redis.zcount('animals', 10, 45))
print(redis.zcard('animals'))
print(redis.zremrangebyrank('animals', 0, 0)) # zremrangebyrank(name, min, max)
print(redis.zremrangebyscore('animals', 200, 250))
print(redis.zrangebyscore('animals', 0, 250))
print(redis.zrangebyscore('animals',0, 250,start=0,num=3,withscores=True))
# 6
# 1
# 3.0
# 2
# 2
# [b'bear', b'Chicken', b'Rabbit', b'Dog']
# [b'bear']
# 3
# 5
# 1
# 1
# [(b'Dog', 10.0), (b'Rabbit', 15.0), (b'Chicken', 45.0)]

Hash

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
print(redis.hset('exotic_fruits', 'pineapple', 2))
print(redis.hsetnx('exotic_fruits', 'kiwi', 5)) # set if not exists
print(redis.hget('exotic_fruits', 'pineapple'))
print(redis.hmget('exotic_fruits', ['papaya', 'pineapple']))
print(redis.hmset('exotic_fruits', {'persimmon': 2, 'litchi': 1,'jujube':7}))
print(redis.hincrby('exotic_fruits', 'papaya', 3)) # hincrby(name, key, amount=1)
print(redis.hexists('exotic_fruits', 'persimmon'))
print(redis.hdel('exotic_fruits', 'persimmon'))
print(redis.hlen('exotic_fruits'))
print(redis.hkeys('exotic_fruits'))
print(redis.hvals('exotic_fruits'))
print(redis.hgetall('exotic_fruits'))
# 1
# 1
# b'2'
# [None, b'2']
# True
# 3
# True
# 1
# 5
# [b'pineapple', b'kiwi', b'litchi', b'jujube', b'papaya']
# [b'2', b'5', b'1', b'7', b'3']
# {b'pineapple': b'2', b'kiwi': b'5', b'litchi': b'1', b'jujube': b'7', b'papaya': b'3'}

CSV

1
2
3
4
5
6
7
8
9
pool = ConnectionPool(host=HOST, password=PASS, port=PORT, db=3)
redis = StrictRedis(connection_pool=pool)

csv_file = 'D:/Program Files/R-3.4.3/library/knitr/examples/download_count.csv'
with open(csv_file) as f:
for line_num, line in enumerate(f, 1):
line = line.strip()
cache_key = '%s_%s' % ('csv', line_num)
redis.lpush(cache_key, line)

PostgreSQL

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
## init.bat
'''
set PGHOME=D:\Bio\Postgre\bin
set PATH=%PGHOME%\bin;%path%
set PGHOST=localhost
set PGLIB=%PGHOME%\lib
set PGDATA=%PGHOME%\data
'''
initdb
pg_ctl start
createuser -h 127.0.0.1 lysql
createdb -h 127.0.0.1 demo
psql -h 127.0.0.1 -U lysql -d demo -p 5432
alter user lysql with password '123456';

## then you can use pgAdmin 4

Connect

1
2
3
4
5
import psycopg2

conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432")

print("Database Opened successfully")
Database Opened successfully

Creat Table

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432")
try:
cursor = conn.cursor()
transaction = cursor.execute("""CREATE TABLE STUDENTS(
ID INT PRIMARY KEY NOT NULL,
NAME TEXT NOT NULL,
AGE INT NOT NULL,
ADDRESS CHAR(50),
PHONE TEXT NOT NULL);
""")
print('Successful')
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
print(conn)
if conn is not None:
conn.close()
Successful
<connection object at 0x04424880; dsn: 'user=lysql password=xxx dbname=demo host=127.0.0.1 port=5432', closed: 0>

Insert Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432")
try:
cursor = conn.cursor()
cursor.execute("INSERT INTO STUDENTS (ID,NAME,AGE,ADDRESS,PHONE) \
VALUES (1, 'Jessica', 22, 'New York', 5044849748 )");
cursor.execute("INSERT INTO STUDENTS (ID,NAME,AGE,ADDRESS,PHONE) \
VALUES (2, 'Lucifier', 15, 'Los Angeles ', 4065248236)");
cursor.execute("INSERT INTO STUDENTS (ID,NAME,AGE,ADDRESS,PHONE) \
VALUES (3, 'Antonio', 13, 'Philadelphia ', 5107449508 )");
cursor.execute("INSERT INTO STUDENTS (ID,NAME,AGE,ADDRESS,PHONE) \
VALUES (4, 'Athena', 25, 'San Francisco ', 3367254545)");
conn.commit()
print ("Records created successfully");
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
Records created successfully

postgresql

Fetch Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432" )
try:
cursor = conn.cursor()
cursor.execute("SELECT * FROM public.students \
ORDER BY id DESC ");
conn.commit()
cursor.fetchone()
cursor.fetchall()
print ("Records fetched successfully");
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
(4,
 'Athena',
 25,
 'San Francisco                                     ',
 '3367254545')






​ [(3,
​ ‘Antonio’,
​ 13,
​ 'Philadelphia ',
​ ‘5107449508’),
​ (2,
​ ‘Lucifier’,
​ 15,
​ 'Los Angeles ',
​ ‘4065248236’),
​ (1,
​ ‘Jessica’,
​ 22,
​ 'New York ',
​ ‘5044849748’)]



​ Records fetched successfully

Update Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432" )
try:
cursor = conn.cursor()
cursor.execute("UPDATE STUDENTS set ADDRESS = 'Seattle' where ID=2 ");
conn.commit()
cursor.execute("SELECT * from STUDENTS ORDER BY ID")
rows = cursor.fetchall()
print("Records fetched successfully")
for row in rows:
print("ID = ", row[0])
print( "NAME = ", row[1])
print( "ADDRESS = ", row[2])
print( "PHONE = ", row[3], "\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
Records fetched successfully
ID =  1
NAME =  Jessica
ADDRESS =  22
PHONE =  New York                                           

ID =  2
NAME =  Lucifier
ADDRESS =  15
PHONE =  Seattle                                            

ID =  3
NAME =  Antonio
ADDRESS =  13
PHONE =  Philadelphia                                       

ID =  4
NAME =  Athena
ADDRESS =  25
PHONE =  San Francisco                                      

Delete Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
conn = psycopg2.connect(database="demo", user="lysql", password="123456", host="127.0.0.1", port="5432" )
try:
cursor = conn.cursor()
cursor.execute("DELETE from STUDENTS where ID=2 OR AGE <=20;");
conn.commit()
print(cursor.rowcount)
cursor.execute("SELECT * from STUDENTS ORDER BY ID")
rows = cursor.fetchall()
for row in rows:
print("ID = ", row[0])
print( "NAME = ", row[1])
print( "ADDRESS = ", row[2])
print( "PHONE = ", row[3], "\n")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
2
ID =  1
NAME =  Jessica
ADDRESS =  22
PHONE =  New York                                           

ID =  4
NAME =  Athena
ADDRESS =  25
PHONE =  San Francisco                                      

MySQL

Initialize

1
2
3
4
5
6
7
8
9
10
## Windows
mysqld --install
mysqld --initialize --user=root --console
# root@localhost: KoL_gm2zn8gl
net start mysql
mysql -u root -p
set password=password('123456');
exit
Drop database Students;
# net stop mysql

Creat db

1
2
3
4
5
6
7
8
9
10
import pymysql

db = pymysql.connect(host='localhost',user='root', password='123456', port=3306)
cursor = db.cursor()
cursor.execute('SELECT VERSION()')
data = cursor.fetchone()
print('Database version:', data)
# Database version: ('5.7.21',)
cursor.execute("CREATE DATABASE Students DEFAULT CHARACTER SET utf8")
db.close()
Database version: ('5.7.21',)

Creat table

1
2
3
4
5
db = pymysql.connect(host='localhost', user='root', password='123456', port=3306, db='Students')
cursor = db.cursor()
sql = 'CREATE TABLE IF NOT EXISTS Demo (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL, age INT NOT NULL, PRIMARY KEY (id))'
cursor.execute(sql)
db.close()

Insert Data

Using Dict

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
db = pymysql.connect(host='localhost', user='root',password='123456', port=3306, db='Students')
data = {
'id': '20143020588',
'name': 'Jessica',
'age': 22
}
table = 'Demo'
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))
sql = 'INSERT INTO {table}({keys}) VALUES ({values})'.format(table=table, keys=keys, values=values)
print(sql)
# INSERT INTO students(id, name, age) VALUES (%s, %s, %s)
try:
cursor = db.cursor()
if cursor.execute(sql, tuple(data.values())):
print('Successful')
db.commit()
except:
print('Failed')
db.rollback() # ACID
db.close()
INSERT INTO Demo(id, name, age) VALUES (%s, %s, %s)
Successful

Update Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
db = pymysql.connect(host='localhost', user='root',password='123456', port=3306, db='Students')
data = {
'id': '20143020200',
'name': 'Jessica',
'age': 21
}
table = 'Demo'
keys = ', '.join(data.keys())
values = ', '.join(['%s'] * len(data))

sql = 'INSERT INTO {table}({keys}) VALUES ({values}) ON DUPLICATE KEY UPDATE'.format(table=table, keys=keys, values=values)
# INSERT INTO Students(id, name, age) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE
update = ','.join([" {key} = %s".format(key=key) for key in data])
sql += update
print(sql)
# INSERT INTO Students(id, name, age) VALUES (%s, %s, %s)
# ON DUPLICATE KEY UPDATE id = %s, name = %s, age = %s
try:
cursor = db.cursor()
if cursor.execute(sql, tuple(data.values())*2):
print('Successful')
db.commit()
except:
print('Failed')
db.rollback()
db.close()
# data = {
# 'id': '20143020201',
# 'name': 'Lucifer',
# 'age': 22
# }
INSERT INTO Demo(id, name, age) VALUES (%s, %s, %s) ON DUPLICATE KEY UPDATE id = %s, name = %s, age = %s
Successful

Delete Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
db = pymysql.connect(host='localhost', user='root',password='123456', port=3306, db='Students')
table = 'Demo'
condition = 'age < 20'

sql = 'DELETE FROM {table} WHERE {condition}'.format(table=table, condition=condition)
try:
cursor = db.cursor()
cursor.execute(sql)
db.commit()
print('Successful')
except:
print('Error')
db.rollback()

db.close()
Successful

Query Data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
db = pymysql.connect(host='localhost', user='root',password='123456', port=3306, db='Students')
table = 'Demo'
condition = 'age > 20'
sql = sql = 'SELECT * FROM {table} WHERE {condition}'.format(table=table, condition=condition)

try:
cursor = db.cursor()
cursor.execute(sql)
print('Count:', cursor.rowcount)
one = cursor.fetchone()
print('One:', one)
results = cursor.fetchall()
print('Results:', results)
print('Results Type:', type(results))
for row in results:
print(row)
except:
print('Error')

# Count: 3
# One: ('20143020200', 'Jessica', 21)
# Results: (('20143020201', 'Lucifer', 22), ('20143020588', 'Jessica', 22))
# Results Type: <class 'tuple'>
# ('20143020201', 'Lucifer', 22)
# ('20143020588', 'Jessica', 22)
Count: 3
One: ('20143020200', 'Jessica', 21)
Results: (('20143020201', 'Lucifer', 22), ('20143020588', 'Jessica', 22))
Results Type: <class 'tuple'>
('20143020201', 'Lucifer', 22)
('20143020588', 'Jessica', 22)

REFERENCES

Redis