When the server was first created, it was easy to manage later , Mainly MySQL Not for me , Cross platform use , A package still has to be packed , So I did it myself
This is the database code of the server I wrote , It can be seen at a glance , The database only exists in a single file data.json in , I/O Very often , The user information file is stored in the running memory , Fast in the case of small data , But when the data is stored to a certain extent , Performance falls precipitously , And stay taskmgr( Task manager ) In one fell swoop, the memory exceeds 1.78G Of pycharm, Be the first .
How to improve performance :
Variable usage set
– variable , A disorderly , Set of elements that do not repeat ( Do not appear list,set,bytearray Equal to none __hash__(self) Hash value class )
– set Of The time complexity is O1, list Of The time complexity is On.
– set Internal storage elements must be removable hash Of , And it is not repeatable .
When every set Each element in has an independent hash The coding , Although the elements are out of order from the outside , But the inside is hash Sorting of codes , When the runtime is querying by encoding , So it will be so fast (warning: It is possible that hash Conflict , But rarely )
Avoid frequent single file calls I/O
Users create folders , A folder corresponds to a user's md5 value （sha256 Any of them will do ）, This is to avoid illegal characters when creating folders .
Similar to file transfer server , The documents sent are the best decompression , Split slices
Only save the user name in the running memory , Save a space , Generally, the database will not be larger than a few TB, Put the code , Registration time these miscellaneous items are placed under the folder
All right , There is no need to look at the back , Press the exit key in the upper left corner to exit
This is a data.py Code ( All data.py stay gitcode - https://gitcode.net/m0_60394896/python/-/blob/05267ff4f3c2267954dc87e505d034229eaa0f98/server/data.py)
from json import load, dump
from os import path, mkdir
from hashlib import md5
from time import time
def encode(data: str):
m = md5()
m.update(data.encode('utf8'))
return m.hexdigest()
file = '.\clients\data.json'
folder = '.\clients'
if not path.exists(folder):
mkdir(folder)
class data:
def __init__(self):
if path.exists(file):
with open(file, 'r') as f:
self.data = load(f)
else:
self.data = {
}
def __get__(self, username, default=None) -> tuple:
return self.data.get(username, default)
def __in__(self, username) -> bool:
return username in self.data.keys()
def __write__(self) -> None:
with open(file, 'w') as f:
dump(self.data, f, indent=4)
def __register__(self, username, password, time: (int, float) = time()) -> None:
...
self.__write__()
def __login__(self, username, password) -> bool:
return self.data[username][0] == encode(password)
def get_time(self, username):
return self.data[username][1]
def handler(self, type: int, username: str, password: str):
...
TCP Chat + Transfer file server socket v2.6
All version records :
v1.0: TCP Chat server socket |PyQt5+socket(TCP Port mapping + Port release )+logging+Thread( Including log ,html)+anaconda pack 32 position exe(3.4 swastika )|python Higher orderv1.1: python TCP Socket server v1.1- New server command function and modification bug(socket+PyQt5)v1.2: python TCP The server v1.2 - Login and registration of new users on the server (json, md5 encryption )v1.3: python TCP The server v1.3 - Server compression test and socket closing processingv1.4: python TCP The server v1.4 - The client connection to the server is abnormal ( Classification of abnormal conditions ) Handlev1.5: PyQt5 Edit drop-down box (comboBox):editable - python TCP The server v1.5 - Add custom parameters to the client connection interface ( Set timeout , The connection address is optional )v1.6: Python TCP The server v1.6 - multiprocessing Multi process and Ctrl-c(SIGINT) sign outv1.7: Python TCP The server v1.7 - PyQt5 server Server comingv1.8: python TCP The server v1.8 - PyQt5 Login interface beautification + Fade in and outv1.9: socketTCP Process documents + Information transmission - TCP Chat file server v1.9 - An epoch-making version update (4.6 swastika )v2.0: TCP Chat file server v2.0 - major bug Repair +PyQt5 File transfer visualizationv2.1: TCP Chat file server v2.1 - Server thread management (threading.enumerate)v2.2: TCP Chat file server v2.2 - Server client socket solution subcontracting / Stick package problem - SocketQueue Inherit and reduce redundancyv2.3: gzip Use - TCP Chat file server v2.3 - File transfer establishes cache system and .gz Decompression of / Compression solves the problem that the running memory is too largev2.4: Network transmission speed measurement - TCP Chat + Transfer file server socket v2.4 - socket Process file transfer speed measurementv2.5: TCP Chat + Transfer file server socket v2.5 - socket The speed measurement specification has been gzip Abandonment ofv2.6: TCP Chat + Transfer file server socket v2.6 - Login registration interface update - loading Interface application
Add database When the user logs in and registers, they are still v1.2,
import itertools
from threading import Thread
def threading(Daemon, name=None, **kwargs):
thread = Thread(**kwargs)
thread.setDaemon(Daemon)
if name:
thread.setName(name)
thread.start()
return thread
data = data()
print(data.handler(0, "tqm", "asdf"))
a = 0
for u in itertools.product("qwertyuiop[]asdfghjkl;'\\zxcvbnm,./`12345678900-", repeat=6):
p = " Stephen asdf"
a += 1
if a % 400 == 0:
data.handler(1, "".join(u), "".join(p), _show_detail=True)
print(a)
else:
data.handler(1, "".join(u), "".join(p), _show_detail=False)
if a > 1000000:
sys.exit(a)
2022-06-20 12:49:00,951 - data.py[line:51] - INFO: Execute the function User handle, timeit 0.000
(False, ' The user doesn't exist !', '')
2022-06-20 12:49:01,743 - data.py[line:51] - INFO:
Execute the function User handle, timeit 0.002
2022-06-20 19:11:20,572 - data.py[line:145] - INFO: size: 799.9Mb(8319541 bytes)
... ...
2022-06-20 19:14:37,143 - data.py[line:51] - INFO: Execute the function User handle, timeit 2.496
2022-06-20 19:14:37,144 - data.py[line:145] - INFO: size: 800.0Mb(8354830 bytes)
2022-06-20 19:17:45,165 - data.py[line:51] - INFO: Execute the function User handle, timeit 2.531
2022-06-20 19:17:45,165 - data.py[line:145] - INFO: size: 800.0Mb(8385887 bytes)
After running for 3, 8400 After iterations , Even a registered user has exceeded the unit of seconds .
from json import load, dump, decoder
from os import path, mkdir, listdir
from hashlib import md5
from time import time
import logging
logging.basicConfig(level=logging.DEBUG,
format="%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
logger = logging.getLogger(__name__)
def _mkdir(_folder):
try:
if not path.exists(_folder):
mkdir(_folder)
return True
except NotImplementedError:
logger.exception("")
return False
def encode(_data: str):
m = md5()
m.update(_data.encode('utf8'))
return m.hexdigest()
def timeit(objname: str, ign=True):
def setup_function(func_name):
def _exec_function(*args, _show_detail=True, **kwargs):
startime = time()
_resp = func_name(*args, **kwargs)
if _show_detail:
logger.info("Execute the function %s%s, timeit %0.3f" % (
objname.title(), "" if ign else f" (at {
str(func_name)})", time() - startime))
return _resp
return _exec_function
return setup_function
folder = r'.\clients'
_mkdir(folder)
data = set()
if path.isdir(folder):
try:
data = set(listdir(folder))
except decoder.JSONDecodeError:
pass
def __in__(username) -> bool:
return username in data
def register(username, password, register_time: (int, float) = time()) -> None:
global data
data.add(username)
user_path = path.join(folder, encode(username))
print(user_path, _mkdir(user_path))
_mkdir(user_path)
with open(path.join(user_path, "user.json"), "w") as f:
dump({
"username": username, "password": password, "register_time": int(register_time)}, f, indent=4)
def login(username, password) -> bool:
with open(path.join(path.join(folder, encode(username)), "user.json"), "r") as f:
_data = load(f)
return _data.get("username", "") == username and _data.get("password", "") == password
def get_time(username):
with open(path.join(path.join(folder, encode(username)), "user.json"), "r") as f:
return load(f).get("register_time")
@timeit("User Handle")
def handler(_type: int, username: str, password: str):
username = username.strip()
if not username:
return False, " User name is not filled !", ""
password = password.strip()
if not password:
return False, " Password not filled in !", ""
if not 2 <= len(username) <= 12:
return False, " The user name must be in 2~12 Between bits !", ""
if not 4 <= len(password) <= 10:
return False, " Password needs to be in 4~10 Between bits !", ""
if _type == 0: # login
if not __in__(username):
return False, " The user doesn't exist !", ""
if not login(username, password):
return False, " user name / Wrong password !", ""
return True, " welcome back , " + username, username
elif _type == 1: # register
if __in__(username):
return False, " There is already a user !", ""
register(username, password)
return True, " Newly arrived , " + username, username
The user name of each user md5 Values are stored in the folder , user.json Release information
