Introduce
Share
Common things in programing...
VIETNAM'S PHONE_REGEX
'^(0[3|5|7|8|9])+([0-9]{8})$'
VN BANKS
from django.utils.translation import gettext as _
list_bank_trans_vi = (
(("ABB"), ("An Binh Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần An Bình")),
(("ACB"), ("Asia Commercial Joint-stock Bank - Ngân hàng thương mại cổ phần Á châu")),
(("AGRIBANK"), (
"Vietnam Bank for Agriculture and Rural Development - Agribank - Ngân hàng nông nghiệp và phát triển nông thông Việt Nam")),
(("BACABANK"), ("Bac A Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Bắc Á")),
(("BID"), (
"Joint Stock Commercial Bank for Investment and Development of Vietnam - Ngân hàng thương mại cổ phần đầu tư và phát triển Việt Nam")),
(("CTG"), (
"Vietnam Joint-Stock Commercial Bank for Industry and Trade - Ngân hàng thương mại cổ phần công thương Việt Nam")),
(("EIB"),
("Vietnam Export Import Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần xuất nhập khẩu Việt Nam")),
(("HDBANK"),
("Ho Chi Minh City Development Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần phát triển TP.HCM")),
(("KLB"), ("Kien Long Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Kiên Long")),
(("LIENVIET"), ("Lien Viet Post Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần bưu điện Liên Việt")),
(("MBB"), ("Military Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quân đội")),
(("MSB"), ("Vietnam Maritime Commercial Stock Bank - Ngân hàng thương mại cổ phần hàng hải Việt Nam")),
(("NAMA"), ("Nam A Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Nam Á")),
(("NCB"), ("National Citizen Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quốc dân")),
(("OCB"), (
"Orient Commercial Joint Stock Bank-Ngan Hang Thuong Mai Co Phan Phuong Dong - Ngân hàng thương mại cổ phần Phương Đông")),
(("PGBANK"),
("Petrolimex Group Commercial Joint Stock Bank (The) - Ngân hàng thương mại cổ phần xăng dầu PETROLIMEX")),
(("PVCOMBANK"), ("Vietnam Public Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần đại chúng Việt Nam")),
(("SCB"), ("Sai Gon Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần Sài Gòn")),
(("SEABANK"), ("Southeast Asia Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Đông Nam Á")),
(("SGB"), ("Saigon Bank for Industry and Trade - Ngân hàng thương mại cổ phần Sài Gòn công thương")),
(("SHB"), ("Saigon - Hanoi Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Sài Gòn - Hà Nội")),
(("STB"),
("Saigon Thuong Tin Commercial Joint-Stock Bank- SACOMBANK - Ngân hàng thương mại cổ phần Sài Gòn thương tín")),
(("TCB"),
("Vietnam Technological and Commercial Joint-Stock Bank - Ngân hàng thương mại cổ phần kỹ thương Việt Nam")),
(("TPB"), ("Tien Phong Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Tiên Phong")),
(("VCB"), (
"Joint Stock Commercial Bank for Foreign Trade of Vietnam- VIETCOMBANK - Ngân hàng thương mại cổ phần ngoại thương Việt Nam")),
(("VIB"), ("VietNam International Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quốc tế Việt Nam")),
(("VIETABANK"), ("Vietnam Asia Commercial Joint - Ngân hàng thương mại cổ phần Việt Á")),
(("VIETCAPITALBANK"), ("Viet Capital Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Bản Việt")),
(("VPB"), ("Vietnam Prosperity Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần Việt Nam thịnh vượng")),
(("VIETBANK"),
("Vietnam Thuong Tin Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Việt Nam Thương Tín")),
)
vn_banks = (
(("ABB"), _("An Binh Commercial Joint Stock Bank")),
(("ACB"), _("Asia Commercial Joint-stock Bank")),
(("AGRIBANK"), _("Vietnam Bank for Agriculture and Rural Development - Agribank")),
(("BACABANK"), _("Bac A Commercial Joint Stock Bank")),
(("BID"), _("Joint Stock Commercial Bank for Investment and Development of Vietnam")),
(("CTG"), _("Vietnam Joint-Stock Commercial Bank for Industry and Trade")),
(("EIB"), _("Vietnam Export Import Commercial Joint Stock Bank")),
(("HDBANK"), _("Ho Chi Minh City Development Joint Stock Commercial Bank")),
(("KLB"), _("Kien Long Commercial Joint Stock Bank")),
(("LIENVIET"), _("Lien Viet Post Joint Stock Commercial Bank")),
(("MBB"), _("Military Commercial Joint Stock Bank")),
(("MSB"), _("Vietnam Maritime Commercial Stock Bank")),
(("NAMA"), _("Nam A Commercial Joint Stock Bank")),
(("NCB"), _("National Citizen Commercial Joint Stock Bank")),
(("OCB"), _("Orient Commercial Joint Stock Bank-Ngan Hang Thuong Mai Co Phan Phuong Dong")),
(("PGBANK"), _("Petrolimex Group Commercial Joint Stock Bank (The)")),
(("PVCOMBANK"), _("Vietnam Public Joint Stock Commercial Bank")),
(("SCB"), _("Sai Gon Joint Stock Commercial Bank")),
(("SEABANK"), _("Southeast Asia Commercial Joint Stock Bank")),
(("SGB"), _("Saigon Bank for Industry and Trade")),
(("SHB"), _("Saigon - Hanoi Commercial Joint Stock Bank")),
(("STB"), _("Saigon Thuong Tin Commercial Joint-Stock Bank- SACOMBANK")),
(("TCB"), _("Vietnam Technological and Commercial Joint-Stock Bank")),
(("TPB"), _("Tien Phong Commercial Joint Stock Bank")),
(("VCB"), _("Joint Stock Commercial Bank for Foreign Trade of Vietnam- VIETCOMBANK")),
(("VIB"), _("VietNam International Commercial Joint Stock Bank")),
(("VIETABANK"), _("Vietnam Asia Commercial Joint")),
(("VIETCAPITALBANK"), _("Viet Capital Commercial Joint Stock Bank")),
(("VPB"), _("Vietnam Prosperity Joint Stock Commercial Bank")),
(("VIETBANK"), _("Vietnam Thuong Tin Commercial Joint Stock Bank")),
)
VIETNAM E-WALLETS
wallet_vn = [
"MoMo",
"ViettelPay",
"ZaloPay",
"VTC Pay ",
"Payoo ",
"ShopeePay",
"VNPay",
"Moca",
"Vimo",
"VinID",
]
There are so many tool can help us to deploy application more easily.
Github-Action
ACT
Github-action does not support locally as gitlab-runner exec ...
like this.
So this tool is play ground at the local.
Thus we can work around before publish with dirty empty commits
git commit --allow-empty -m "Empty-Commit" && git push
That makes sense versioning control, team work.
act -g | l ...
act --help
Ship with input and var && env
#![allow(unused)] fn main() { println!("aaaa"); }
Jenkins
gitlab
Just in case ipv4 does not assign to container.
sudo firewall-cmd --zone=trusted --change-interface=lxdbr0 --permanent
sudo firewall-cmd --reload
Prevent conflict with Docker.
iptables -I DOCKER-USER -i <network_bridge> -o <external_interface> -j ACCEPT
iptables -I DOCKER-USER -o <network_bridge> -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
Docker inside LXC.
lxc storage create tower btrfs
lxc storage volume create tower docker
lxc config device add tower tower disk pool=tower source=docker path=/var/lib/docker
lxc config set tower security.nesting=true security.syscalls.intercept.mknod=true security.syscalls.intercept.setxattr=true
lxc restart tower
Rust
Pythonista
There are so many error caused by datetime. So I decided to write this article.
Determine exact date and time with timedelta
timedelta
does not support year right?
that why we have to do more calculate
Let's use python built-in package calendar
.
import calendar
from datetime import datetime
days = 366 if calendar.isleap(datetime.now().year) else 365
For some details: send email hapy birthday for next year, count down event day, billing stuff, etc...
Everything works fine if this year not leap.
What if I send an email happy new year on the day at the end of the year before?
Python
All we understood this way help us keep away N+1 hit to the db, right?.
We have simple models here.
class Author(models.Model):
name = models.CharField(max_length=255)
class Meta:
db_table = "author"
class Tag(models.Model):
name = models.CharField(max_length=255)
class Meta:
db_table = "tag"
class Book(models.Model):
name = models.CharField(max_length=255)
author = models.ForeignKey(Author, on_delete=models.CASCADE)
tag = models.ManyToManyField(Tag)
class Meta:
db_table = "book"
1. Select related
Now let's do some shell
Book.objects.select_related("author")
SELECT "book"."id",
"book"."name",
"book"."author_id",
"author"."id",
"author"."name"
FROM "book"
INNER JOIN "author"
ON ("book"."author_id" = "author"."id")
LIMIT 21
Execution time: 0.000162s [Database: default]
Book.objects.select_related("tag")
FieldError: Invalid field name(s) given in select_related: 'tag'. Choices are: author
Ooppppps
Even you try
Tag.objects.select_related('book')
That's will give you the same error too.
For the
select_related
we can only use if theForeignKey
involves.
So next is interesting part
2. Prefetch related.
In [21]: Book.objects.prefetch_related("author")
Out[21]:
SELECT "book"."id",
"book"."name",
"book"."author_id"
FROM "book"
LIMIT 21
Execution time: 0.000097s [Database: default]
SELECT "author"."id",
"author"."name"
FROM "author"
WHERE "author"."id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
And now for the tag
In [23]: Book.objects.prefetch_related("tag")
Out[23]:
SELECT "book"."id",
"book"."name",
"book"."author_id"
FROM "book"
LIMIT 21
Execution time: 0.000102s [Database: default]
SELECT ("book_tag"."book_id") AS "_prefetch_related_val_book_id",
"tag"."id",
"tag"."name"
FROM "tag"
INNER JOIN "book_tag"
ON ("tag"."id" = "book_tag"."tag_id")
WHERE "book_tag"."book_id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
Hee it's worked either one of them is not ManyToManyField
. There is no sign of any error as select_related
But very very different statements as you can see and compare.
Wait what.. what is book_tag
? where do you come from?.
It's the table's name auto generated by django.
For the m2m relation we have to have a middle table to map both id's.
We can show the table in our db.
sqlite>
SELECT
name
FROM
sqlite_schema
WHERE
type ='table' AND
name NOT LIKE 'django_%' AND
name NOT LIKE 'auth_%' AND
name NOT LIKE 'sqlite_%';
tag
book
book_tag
SELECT * FROM book_tag;
id book_id tag_id
-- ------- ------
1 1 2
2 1 1
4 1 3
...
That's explained why we got unmatch statement.
We have to do more JOIN
after get the tag's id, instead of go to destination as author
does.
Interested part
But in the Tag's view. How do i get the books contains this tag?.
As we know the Tag
connected to Book
through book_tag
right ?
Tag.objects.filter().prefetch_related('book')
AttributeError: Cannot find 'book' on Tag object, 'book' is an invalid parameter to prefetch_related()
Argggggggggggggggg that's why i expected more than 24hrs for a single day.
Go to the docs as a usual? No!.
In [35]: vars(Tag)
Out[35]:
mappingproxy({'__module__': 'author.models',
'__doc__': 'Tag(id, name)',
'_meta': <Options for Tag>,
'DoesNotExist': author.models.Tag.DoesNotExist,
'MultipleObjectsReturned': author.models.Tag.MultipleObjectsReturned,
'name': <django.db.models.query_utils.DeferredAttribute at 0x7ffa7c952260>,
'id': <django.db.models.query_utils.DeferredAttribute at 0x7ffa7c952380>,
'objects': <django.db.models.manager.ManagerDescriptor at 0x7ffa7c952320>,
'book_set': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7ffa7c953610>})
Did you see that book_set
just apply it.
In [36]: Tag.objects.filter().prefetch_related('book_set')
Out[36]: SELECT "tag"."id",
"tag"."name"
FROM "tag"
LIMIT 21
Execution time: 0.000046s [Database: default]
SELECT ("book_tag"."tag_id") AS "_prefetch_related_val_tag_id",
"book"."id",
"book"."name",
"book"."author_id"
FROM "book"
INNER JOIN "book_tag"
ON ("book"."id" = "book_tag"."book_id")
WHERE "book_tag"."tag_id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
Now the hack is _set
.
Just related_name
at the Python object level, not relate to sql stuff. If you dont set, django will use it as default.
Now I added it.
class Book(models.Model):
name = models.CharField(max_length=255)
author = models.ForeignKey(Author, on_delete=models.CASCADE)
tag = models.ManyToManyField(Tag, related_name='book')
And check it again.
In [3]: vars(Tag)
Out[3]:
mappingproxy({...
'name': <django.db.models.query_utils.DeferredAttribute at 0x7fccd3436c20>,
'id': <django.db.models.query_utils.DeferredAttribute at 0x7fccd3436d40>,
'objects': <django.db.models.manager.ManagerDescriptor at 0x7fccd3436ce0>,
'book': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3436860>})
Tag.objects.filter().prefetch_related('book')
<QuerySet [...]>
Hooray it's work as I expected!
Now let's try with ForeignKey
at the Author
view
Author.objects.filter().prefetch_related('book_set')
<QuerySet [...]>
Works like a champ...
More salt?
If this Tag
table have others relation than Book
like a post, news, event, etc...
Ok let's do it.
i added new model below.
class Post(models.Model):
name = models.CharField(max_length=255)
content = models.TextField()
tag = models.ManyToManyField(Tag)
class Meta:
db_table = "post"
Then run make/migrate.
In [7]: vars(Tag)
Out[7]:
mappingproxy({...
'objects': <django.db.models.manager.ManagerDescriptor at 0x7fccd3436ce0>,
'book': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3436860>,
'post_set': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3435d80>
})
Post.objects.filter().prefetch_related('book_set')
<QuerySet []>
Here it is!
1. SAME KIND.
Base models
# /product/models.py
from django.db import models
class Product(models.Model):
name = models.CharField(max_length=255)
class Variant(models.Model):
name = models.CharField(max_length=255)
product = models.ForeignKey(Product, on_delete=models.CASCADE)
price = models.PositiveIntegerField()
i got my app name is product
and register in INSTALLED_APPS
settings
price = models.PositiveIntegerField()
need to be decimal
there are db schema having some primitive type: char, int, varchar... another.
But for all it's just number and text.
Just simply to change the type to DecimalField
class Variant(models.Model):
name = models.CharField(max_length=255)
product = models.ForeignKey(Product, on_delete=models.CASCADE)
price = models.DecimalField(decimal_places=3, max_digits=999)
and then run make migration
python manage.py makemigrations product
# /product/migrations/0002_alter_variant_price.py
# Generated by Django 4.1.7 on 2023-02-23 22:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("product", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="variant",
name="price",
field=models.DecimalField(decimal_places=3, max_digits=999),
),
]
Django will automate generate a new migration file base on our behavior as you can see above.
And then migrate it's
python manage.py migrate product
Now just find where we implemented this field and do more change logic, syntax, etc...
Everything will work.
Well it's just a simple case, because it is the same type in general: NUMBER
2. NEW FIELD BASED ON EXISTED FIELD.
Now we do harder from int
to datetime
I have a subscribes app like product above too.
# subscribe/models.py
class Subscriber(models.Model):
email = models.EmailField()
age = models.SmallIntegerField()
This model run so long ago and have a thousand records in it
For more logical and painful of code it's should be date of birth(DOB)
I have to migrate existed data to new type right?
class Subscriber(models.Model):
email = models.EmailField()
age = models.SmallIntegerField()
dob = models.DateField()
Well just add a dob field and run make & migrate.
python manage.py makemigrations
It is impossible to add a non-nullable field 'dob' to subscriber without specifying a default. This is because the database needs something to populate existing rows.
Please select a fix:
1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
2) Quit and manually define a default value in models.py.
Select an option:
Ahhhh that it.
If i place null=True
to attribute in the model's field that's will work perfectly within null value. But for what?
So now we have to do more calculate for it.
Make a fresh migrate app from django template.
python manage.py makemigrations subscribe --name convert_age_dob --empty
and the output
Migrations for 'subscribe':
subscribe/migrations/0002_convert_age_dob.py
# subscribe/migrations/0002_convert_age_dob.py
# Generated by Django 4.1.7 on 2023-02-23 23:09
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("subscribe", "0001_initial"),
]
operations = []
Here the battlefield
Need a function convert from age to DOB right ?
from django.utils import timezone
def age_to_dob(apps, schema_editor):
Subscriber = apps.get_model('subscribe', 'Subscriber')
today = timezone.now()
for sub in Subscriber.objects.all():
sub.dob = today.replace(year=today.year-sub.age) # that too dirty but i'll fix it later
sub.save()
and let operation run it
operations = [
migrations.RunPython(age_to_dob, migrations.RunPython.noop),
]
migrations.RunPython.noop
is the step backward if we revert the migration. so for do nothing i added noop
Now my completed migration file is:
# Generated by Django 4.1.7 on 2023-02-23 23:09
from django.db import migrations, models
from django.utils import timezone
def age_to_dob(apps, schema_editor):
Subscriber = apps.get_model('subscribe', 'Subscriber')
today = timezone.now()
for sub in Subscriber.objects.all():
sub.dob = today.replace(year=today.year-sub.age) # that too dirty but i'll fix it later
sub.save()
class Migration(migrations.Migration):
dependencies = [
("subscribe", "0001_initial"),
]
operations = [
migrations.AddField(
model_name='Subscriber',
name='dob',
field=models.DateField(null=True, default=None),
preserve_default=False,
),
migrations.RunPython(age_to_dob, migrations.RunPython.noop),
migrations.AlterField(
model_name='Subscriber',
name='dob',
field=models.DateField(),
preserve_default=True,
),
]
well a little big rigt ?
we have migrations.AddField
anf migrations.AlterField
it's just the trick to bypass not-null value from django migrate schema. cuz we need this always having data
Otherwise, you will get the error django.db.utils.IntegrityError: NOT NULL constraint failed:...
and run migrate this file.
3. SAME FIELD AND DIFFERENT TYPE.
This is the hard part on refactor operation a projects.
Added new field and then copy data in to it will make many critical error on release and if we run incontainer
we have to create lots of releases that's increment time and effort!
As all we know. django model have pk
default for your record, it is a biginterger
and have auto increment
on the back.
Now if we expose this to the public like id of post or user id. This way is the door to invites crawler get my own data. Cuz it's predictable.
So it must be unpredictable, by using uuid or other algorythm to create identify instead.
I got my user model here
from django.contrib.auth.models import AbstractUser
from django.db import models
class User(AbstractUser):
alias = models.CharField(max_length=255)
...
That run quite long ago and got a lot of users\
At this if we call .id
or .pk
will return int for each record.
And tons of code implemented before take time to refactor. Would you add more field like uuid
and then go to fix, add, delete old logic ???
NOOOOOOOOOOOOOOO! imma lazy guy. And i impl like this.
import uuid
from django.contrib.auth.models import AbstractUser
from django.db import models
class User(AbstractUser):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
alias = models.CharField(max_length=255)
...
Then make a function generate new id inside of migration stage. make it.
user/migrations/0002_alter_user_id.py
# Generated by Django 4.1.7 on 2023-02-24 01:14
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("user", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="user",
name="id",
field=models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
]
Django generated it for me. but it just applies to new record which insert in the future. what should I do for existed id.
So i apply my flow here and run it without any error.
from django.db import migrations, models
import uuid
def create_uuid(apps, schema_editor):
User = apps.get_model('user', 'User')
for user in list(User.objects.all()):
user.id = uuid.UUID(int=user.id)
user.save()
class Migration(migrations.Migration):
dependencies = [
("user", "0001_initial"),
]
operations = [
# add new one
migrations.AddField(
model_name='user',
name='uuid',
field=models.UUIDField(null=True),
),
# seed new one
migrations.RunPython(create_uuid, migrations.RunPython.noop),
# change attributes
migrations.AlterField(
model_name='user',
name='uuid',
field=models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
# remove old one
migrations.RemoveField('User', 'id'),
# rename to new
migrations.RenameField(
model_name='User',
old_name='uuid',
new_name='id'
),
# Update attributes
migrations.AlterField(
model_name='user',
name='id',
field=models.UUIDField(
primary_key=True, default=uuid.uuid4, serialize=False, editable=False
),
),
]
Celery
Snippet
Some necessarily need to be added to new project.
Don't have time find on the internet.
I'm too old to remember.
INIT.
gcl [email protected]:bboyadao/django_template.git proj_name \
&& cd proj_name \
&& rm -rf .git
&& git init
poetry install
SETTINGS.
ALLOWED_HOSTS = ["*"]
AUTH_APPS = [
"rest_framework_simplejwt.token_blacklist",
"oauth2_provider",
"rest_framework.authtoken",
"dj_rest_auth",
"dj_rest_auth.registration",
"allauth",
"allauth.account",
"allauth.socialaccount",
"allauth.socialaccount.providers.facebook",
"allauth.socialaccount.providers.google",
]
DOCS_APPS = ["drf_spectacular",
"drf_spectacular_sidecar"]
CELERY_APPS = ["django_celery_results",
"django_celery_beat"]
LIB_APPS = ["rest_framework",
"django_filters",
"phonenumber_field",
"corsheaders",
"fcm_django",
"actstream",
]
EXTERNAL_APPS = LIB_APPS + CELERY_APPS + AUTH_APPS + DOCS_APPS
INTERNAL_APPS = []
INSTALLED_APPS += EXTERNAL_APPS + INTERNAL_APPS