Introduction
![]()
Share
Common things in programming...
VIETNAM'S PHONE_REGEX
'^(0[3|5|7|8|9])+([0-9]{8})$'
VN BANKS
from django.utils.translation import gettext as _
list_bank_trans_vi = (
(("ABB"), ("An Binh Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần An Bình")),
(("ACB"), ("Asia Commercial Joint-stock Bank - Ngân hàng thương mại cổ phần Á châu")),
(("AGRIBANK"), (
"Vietnam Bank for Agriculture and Rural Development - Agribank - Ngân hàng nông nghiệp và phát triển nông thông Việt Nam")),
(("BACABANK"), ("Bac A Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Bắc Á")),
(("BID"), (
"Joint Stock Commercial Bank for Investment and Development of Vietnam - Ngân hàng thương mại cổ phần đầu tư và phát triển Việt Nam")),
(("CTG"), (
"Vietnam Joint-Stock Commercial Bank for Industry and Trade - Ngân hàng thương mại cổ phần công thương Việt Nam")),
(("EIB"),
("Vietnam Export Import Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần xuất nhập khẩu Việt Nam")),
(("HDBANK"),
("Ho Chi Minh City Development Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần phát triển TP.HCM")),
(("KLB"), ("Kien Long Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Kiên Long")),
(("LIENVIET"), ("Lien Viet Post Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần bưu điện Liên Việt")),
(("MBB"), ("Military Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quân đội")),
(("MSB"), ("Vietnam Maritime Commercial Stock Bank - Ngân hàng thương mại cổ phần hàng hải Việt Nam")),
(("NAMA"), ("Nam A Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Nam Á")),
(("NCB"), ("National Citizen Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quốc dân")),
(("OCB"), (
"Orient Commercial Joint Stock Bank-Ngan Hang Thuong Mai Co Phan Phuong Dong - Ngân hàng thương mại cổ phần Phương Đông")),
(("PGBANK"),
("Petrolimex Group Commercial Joint Stock Bank (The) - Ngân hàng thương mại cổ phần xăng dầu PETROLIMEX")),
(("PVCOMBANK"), ("Vietnam Public Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần đại chúng Việt Nam")),
(("SCB"), ("Sai Gon Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần Sài Gòn")),
(("SEABANK"), ("Southeast Asia Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Đông Nam Á")),
(("SGB"), ("Saigon Bank for Industry and Trade - Ngân hàng thương mại cổ phần Sài Gòn công thương")),
(("SHB"), ("Saigon - Hanoi Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Sài Gòn - Hà Nội")),
(("STB"),
("Saigon Thuong Tin Commercial Joint-Stock Bank- SACOMBANK - Ngân hàng thương mại cổ phần Sài Gòn thương tín")),
(("TCB"),
("Vietnam Technological and Commercial Joint-Stock Bank - Ngân hàng thương mại cổ phần kỹ thương Việt Nam")),
(("TPB"), ("Tien Phong Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Tiên Phong")),
(("VCB"), (
"Joint Stock Commercial Bank for Foreign Trade of Vietnam- VIETCOMBANK - Ngân hàng thương mại cổ phần ngoại thương Việt Nam")),
(("VIB"), ("VietNam International Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần quốc tế Việt Nam")),
(("VIETABANK"), ("Vietnam Asia Commercial Joint - Ngân hàng thương mại cổ phần Việt Á")),
(("VIETCAPITALBANK"), ("Viet Capital Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Bản Việt")),
(("VPB"), ("Vietnam Prosperity Joint Stock Commercial Bank - Ngân hàng thương mại cổ phần Việt Nam thịnh vượng")),
(("VIETBANK"),
("Vietnam Thuong Tin Commercial Joint Stock Bank - Ngân hàng thương mại cổ phần Việt Nam Thương Tín")),
)
vn_banks = (
(("ABB"), _("An Binh Commercial Joint Stock Bank")),
(("ACB"), _("Asia Commercial Joint-stock Bank")),
(("AGRIBANK"), _("Vietnam Bank for Agriculture and Rural Development - Agribank")),
(("BACABANK"), _("Bac A Commercial Joint Stock Bank")),
(("BID"), _("Joint Stock Commercial Bank for Investment and Development of Vietnam")),
(("CTG"), _("Vietnam Joint-Stock Commercial Bank for Industry and Trade")),
(("EIB"), _("Vietnam Export Import Commercial Joint Stock Bank")),
(("HDBANK"), _("Ho Chi Minh City Development Joint Stock Commercial Bank")),
(("KLB"), _("Kien Long Commercial Joint Stock Bank")),
(("LIENVIET"), _("Lien Viet Post Joint Stock Commercial Bank")),
(("MBB"), _("Military Commercial Joint Stock Bank")),
(("MSB"), _("Vietnam Maritime Commercial Stock Bank")),
(("NAMA"), _("Nam A Commercial Joint Stock Bank")),
(("NCB"), _("National Citizen Commercial Joint Stock Bank")),
(("OCB"), _("Orient Commercial Joint Stock Bank-Ngan Hang Thuong Mai Co Phan Phuong Dong")),
(("PGBANK"), _("Petrolimex Group Commercial Joint Stock Bank (The)")),
(("PVCOMBANK"), _("Vietnam Public Joint Stock Commercial Bank")),
(("SCB"), _("Sai Gon Joint Stock Commercial Bank")),
(("SEABANK"), _("Southeast Asia Commercial Joint Stock Bank")),
(("SGB"), _("Saigon Bank for Industry and Trade")),
(("SHB"), _("Saigon - Hanoi Commercial Joint Stock Bank")),
(("STB"), _("Saigon Thuong Tin Commercial Joint-Stock Bank- SACOMBANK")),
(("TCB"), _("Vietnam Technological and Commercial Joint-Stock Bank")),
(("TPB"), _("Tien Phong Commercial Joint Stock Bank")),
(("VCB"), _("Joint Stock Commercial Bank for Foreign Trade of Vietnam- VIETCOMBANK")),
(("VIB"), _("VietNam International Commercial Joint Stock Bank")),
(("VIETABANK"), _("Vietnam Asia Commercial Joint")),
(("VIETCAPITALBANK"), _("Viet Capital Commercial Joint Stock Bank")),
(("VPB"), _("Vietnam Prosperity Joint Stock Commercial Bank")),
(("VIETBANK"), _("Vietnam Thuong Tin Commercial Joint Stock Bank")),
)
VIETNAM E-WALLETS
wallet_vn = [
"MoMo",
"ViettelPay",
"ZaloPay",
"VTC Pay ",
"Payoo ",
"ShopeePay",
"VNPay",
"Moca",
"Vimo",
"VinID",
]
There are so many tools that can help us deploy applications more easily.
Github-Action
ACT
GitHub Actions does not support local execution like gitlab-runner exec ... here.
So this tool is a playground for local development.
Thus we can work around the issue before publishing with dirty empty commits
git commit --allow-empty -m "Empty-Commit" && git push
This makes sense for version control and teamwork.
act -g | l ...
act --help
Ship with input and var && env
#![allow(unused)] fn main() { println!("aaaa"); }
Jenkins
gitlab
Just in case IPv4 is not assigned to the container.
sudo firewall-cmd --zone=trusted --change-interface=lxdbr0 --permanent
sudo firewall-cmd --reload
Prevent conflict with Docker.
iptables -I DOCKER-USER -i <network_bridge> -o <external_interface> -j ACCEPT
iptables -I DOCKER-USER -o <network_bridge> -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
Docker inside LXC.
lxc storage create tower btrfs
lxc storage volume create tower docker
lxc config device add tower tower disk pool=tower source=docker path=/var/lib/docker
lxc config set tower security.nesting=true security.syscalls.intercept.mknod=true security.syscalls.intercept.setxattr=true
lxc restart tower
Rust
Pythonista
There are so many errors caused by datetime. So I decided to write this article.
Determining the exact date and time with timedelta
timedelta does not support years, right?
That's why we have to do more calculations.
Let's use Python's built-in package calendar.
import calendar
from datetime import datetime
days = 366 if calendar.isleap(datetime.now().year) else 365
For some details: send a happy birthday email for next year, count down to event day, billing stuff, etc.
Everything works fine if this year is not a leap year.
What if I send a happy new year email on the day at the end of the year before?
Python
We all understand this approach helps us avoid N+1 hits to the database, right?
We have simple models here.
class Author(models.Model):
name = models.CharField(max_length=255)
class Meta:
db_table = "author"
class Tag(models.Model):
name = models.CharField(max_length=255)
class Meta:
db_table = "tag"
class Book(models.Model):
name = models.CharField(max_length=255)
author = models.ForeignKey(Author, on_delete=models.CASCADE)
tag = models.ManyToManyField(Tag)
class Meta:
db_table = "book"
1. Select related
Now let's do some shell
Book.objects.select_related("author")
SELECT "book"."id",
"book"."name",
"book"."author_id",
"author"."id",
"author"."name"
FROM "book"
INNER JOIN "author"
ON ("book"."author_id" = "author"."id")
LIMIT 21
Execution time: 0.000162s [Database: default]
Book.objects.select_related("tag")
FieldError: Invalid field name(s) given in select_related: 'tag'. Choices are: author
Oops!
Even if you try
Tag.objects.select_related('book')
That will give you the same error too.
For
select_related, we can only use it if aForeignKeyis involved.
So the next part is interesting
2. Prefetch related.
In [21]: Book.objects.prefetch_related("author")
Out[21]:
SELECT "book"."id",
"book"."name",
"book"."author_id"
FROM "book"
LIMIT 21
Execution time: 0.000097s [Database: default]
SELECT "author"."id",
"author"."name"
FROM "author"
WHERE "author"."id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
And now for the tag
In [23]: Book.objects.prefetch_related("tag")
Out[23]:
SELECT "book"."id",
"book"."name",
"book"."author_id"
FROM "book"
LIMIT 21
Execution time: 0.000102s [Database: default]
SELECT ("book_tag"."book_id") AS "_prefetch_related_val_book_id",
"tag"."id",
"tag"."name"
FROM "tag"
INNER JOIN "book_tag"
ON ("tag"."id" = "book_tag"."tag_id")
WHERE "book_tag"."book_id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
Hey, it worked even though one of them is not ManyToManyField. There is no sign of any error like with select_related.
But the statements are very, very different as you can see and compare.
Wait, what... what is book_tag? Where do you come from?
It's the table name auto-generated by Django.
For the m2m relation, we have to have a middle table to map both IDs.
We can show the table in our db.
sqlite>
SELECT
name
FROM
sqlite_schema
WHERE
type ='table' AND
name NOT LIKE 'django_%' AND
name NOT LIKE 'auth_%' AND
name NOT LIKE 'sqlite_%';
tag
book
book_tag
SELECT * FROM book_tag;
id book_id tag_id
-- ------- ------
1 1 2
2 1 1
4 1 3
...
That explains why we got unmatched statements.
We have to do more JOINs after getting the tag's ID, instead of going directly to the destination as author does.
Interesting part
But from the Tag's perspective, how do I get the books that contain this tag?
As we know, the Tag is connected to Book through book_tag, right?
Tag.objects.filter().prefetch_related('book')
AttributeError: Cannot find 'book' on Tag object, 'book' is an invalid parameter to prefetch_related()
Argggggggggggggggg that's why I wish there were more than 24 hours in a single day.
Go to the docs as usual? No!
In [35]: vars(Tag)
Out[35]:
mappingproxy({'__module__': 'author.models',
'__doc__': 'Tag(id, name)',
'_meta': <Options for Tag>,
'DoesNotExist': author.models.Tag.DoesNotExist,
'MultipleObjectsReturned': author.models.Tag.MultipleObjectsReturned,
'name': <django.db.models.query_utils.DeferredAttribute at 0x7ffa7c952260>,
'id': <django.db.models.query_utils.DeferredAttribute at 0x7ffa7c952380>,
'objects': <django.db.models.manager.ManagerDescriptor at 0x7ffa7c952320>,
'book_set': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7ffa7c953610>})
Did you see that book_set just apply it.
In [36]: Tag.objects.filter().prefetch_related('book_set')
Out[36]: SELECT "tag"."id",
"tag"."name"
FROM "tag"
LIMIT 21
Execution time: 0.000046s [Database: default]
SELECT ("book_tag"."tag_id") AS "_prefetch_related_val_tag_id",
"book"."id",
"book"."name",
"book"."author_id"
FROM "book"
INNER JOIN "book_tag"
ON ("book"."id" = "book_tag"."book_id")
WHERE "book_tag"."tag_id" IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21)
Now the hack is _set.
It's just related_name at the Python object level, not related to SQL stuff. If you don't set it, Django will use it as the default.
Now I added it.
class Book(models.Model):
name = models.CharField(max_length=255)
author = models.ForeignKey(Author, on_delete=models.CASCADE)
tag = models.ManyToManyField(Tag, related_name='book')
And check it again.
In [3]: vars(Tag)
Out[3]:
mappingproxy({...
'name': <django.db.models.query_utils.DeferredAttribute at 0x7fccd3436c20>,
'id': <django.db.models.query_utils.DeferredAttribute at 0x7fccd3436d40>,
'objects': <django.db.models.manager.ManagerDescriptor at 0x7fccd3436ce0>,
'book': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3436860>})
Tag.objects.filter().prefetch_related('book')
<QuerySet [...]>
Hooray, it works as I expected!
Now let's try with ForeignKey at the Author view
Author.objects.filter().prefetch_related('book_set')
<QuerySet [...]>
Works like a champ...
Want more?
If this Tag table has other relations besides Book, like post, news, event, etc.
OK, let's do it.
I added a new model below.
class Post(models.Model):
name = models.CharField(max_length=255)
content = models.TextField()
tag = models.ManyToManyField(Tag)
class Meta:
db_table = "post"
Then run make/migrate.
In [7]: vars(Tag)
Out[7]:
mappingproxy({...
'objects': <django.db.models.manager.ManagerDescriptor at 0x7fccd3436ce0>,
'book': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3436860>,
'post_set': <django.db.models.fields.related_descriptors.ManyToManyDescriptor at 0x7fccd3435d80>
})
Post.objects.filter().prefetch_related('book_set')
<QuerySet []>
Here it is!
1. SAME KIND.
Base models
# /product/models.py
from django.db import models
class Product(models.Model):
name = models.CharField(max_length=255)
class Variant(models.Model):
name = models.CharField(max_length=255)
product = models.ForeignKey(Product, on_delete=models.CASCADE)
price = models.PositiveIntegerField()
I have an app named product registered in INSTALLED_APPS settings.
price = models.PositiveIntegerField() needs to be decimal.
There are database schemas having primitive types: char, int, varchar, etc.
But in the end, it's just numbers and text.
Simply change the type to DecimalField
class Variant(models.Model):
name = models.CharField(max_length=255)
product = models.ForeignKey(Product, on_delete=models.CASCADE)
price = models.DecimalField(decimal_places=3, max_digits=999)
and then run makemigrations
python manage.py makemigrations product
# /product/migrations/0002_alter_variant_price.py
# Generated by Django 4.1.7 on 2023-02-23 22:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("product", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="variant",
name="price",
field=models.DecimalField(decimal_places=3, max_digits=999),
),
]
Django will automatically generate a new migration file based on our changes as you can see above.
And then migrate it
python manage.py migrate product
Now just find where we implemented this field and make additional logic changes, syntax updates, etc.
Everything will work.
Well, it's just a simple case because it is the same type in general: NUMBER
2. NEW FIELD BASED ON EXISTING FIELD.
Now let's do something harder: migrating from int to datetime.
I have a subscribes app similar to the product example above.
# subscribe/models.py
class Subscriber(models.Model):
email = models.EmailField()
age = models.SmallIntegerField()
This model has been running for a long time and has thousands of records in it.
For better logic and to reduce code complexity, it should be date of birth (DOB).
I have to migrate existing data to the new type, right?
class Subscriber(models.Model):
email = models.EmailField()
age = models.SmallIntegerField()
dob = models.DateField()
Well, just add a dob field and run makemigrations & migrate.
python manage.py makemigrations
It is impossible to add a non-nullable field 'dob' to subscriber without specifying a default. This is because the database needs something to populate existing rows.
Please select a fix:
1) Provide a one-off default now (will be set on all existing rows with a null value for this column)
2) Quit and manually define a default value in models.py.
Select an option:
Ahhhh, that's it.
If I set null=True as an attribute in the model's field, it will work perfectly with null values. But what's the point?
So now we have to do more calculations for it.
Create an empty migration from the Django template.
python manage.py makemigrations subscribe --name convert_age_dob --empty
and the output
Migrations for 'subscribe':
subscribe/migrations/0002_convert_age_dob.py
# subscribe/migrations/0002_convert_age_dob.py
# Generated by Django 4.1.7 on 2023-02-23 23:09
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("subscribe", "0001_initial"),
]
operations = []
Here's the battlefield.
We need a function to convert from age to DOB, right?
from django.utils import timezone
def age_to_dob(apps, schema_editor):
Subscriber = apps.get_model('subscribe', 'Subscriber')
today = timezone.now()
for sub in Subscriber.objects.all():
sub.dob = today.replace(year=today.year-sub.age) # that's too dirty but I'll fix it later
sub.save()
and let operation run it
operations = [
migrations.RunPython(age_to_dob, migrations.RunPython.noop),
]
migrations.RunPython.noop is the backward step if we revert the migration. So to do nothing, I added noop.
Now my completed migration file is:
# Generated by Django 4.1.7 on 2023-02-23 23:09
from django.db import migrations, models
from django.utils import timezone
def age_to_dob(apps, schema_editor):
Subscriber = apps.get_model('subscribe', 'Subscriber')
today = timezone.now()
for sub in Subscriber.objects.all():
sub.dob = today.replace(year=today.year-sub.age) # that's too dirty but I'll fix it later
sub.save()
class Migration(migrations.Migration):
dependencies = [
("subscribe", "0001_initial"),
]
operations = [
migrations.AddField(
model_name='Subscriber',
name='dob',
field=models.DateField(null=True, default=None),
preserve_default=False,
),
migrations.RunPython(age_to_dob, migrations.RunPython.noop),
migrations.AlterField(
model_name='Subscriber',
name='dob',
field=models.DateField(),
preserve_default=True,
),
]
Well, a little big, right?
We have migrations.AddField and migrations.AlterField—it's just a trick to bypass the not-null constraint from Django's migration schema, because we need this field to always have data.
Otherwise, you will get the error django.db.utils.IntegrityError: NOT NULL constraint failed:...
Then run migrate on this file.
3. SAME FIELD AND DIFFERENT TYPE.
This is the hard part of refactoring a project.
Adding a new field and then copying data into it will cause many critical errors on release, and if we run incontainerswe have to create lots of releases, which increases time and effort!
As we all know, Django models have a pk by default for your records—it is a biginteger with auto increment on the backend.
Now if we expose this to the public like a post ID or user ID, this is a door that invites crawlers to get our data because it's predictable.
So it must be unpredictable by using UUID or another algorithm to create identifiers instead.
I have my user model here
from django.contrib.auth.models import AbstractUser
from django.db import models
class User(AbstractUser):
alias = models.CharField(max_length=255)
...
This has been running for quite a long time and has a lot of users.
At this point, if we call .id or .pk, it will return an int for each record.
And there are tons of code implemented previously that would take time to refactor. Would you add another field like uuid and then go fix, add, and delete old logic???
NOOOOOOOOOOOOOOO! I'm a lazy guy. So I implemented it like this.
import uuid
from django.contrib.auth.models import AbstractUser
from django.db import models
class User(AbstractUser):
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
alias = models.CharField(max_length=255)
...
Then make a function generate new id inside of migration stage. make it.
user/migrations/0002_alter_user_id.py
# Generated by Django 4.1.7 on 2023-02-24 01:14
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
("user", "0001_initial"),
]
operations = [
migrations.AlterField(
model_name="user",
name="id",
field=models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
]
Django generated it for me, but it only applies to new records that will be inserted in the future. What should I do for existing IDs?
So I applied my workflow here and ran it without any errors.
from django.db import migrations, models
import uuid
def create_uuid(apps, schema_editor):
User = apps.get_model('user', 'User')
for user in list(User.objects.all()):
user.id = uuid.UUID(int=user.id)
user.save()
class Migration(migrations.Migration):
dependencies = [
("user", "0001_initial"),
]
operations = [
# add new one
migrations.AddField(
model_name='user',
name='uuid',
field=models.UUIDField(null=True),
),
# seed new one
migrations.RunPython(create_uuid, migrations.RunPython.noop),
# change attributes
migrations.AlterField(
model_name='user',
name='uuid',
field=models.UUIDField(
default=uuid.uuid4, editable=False, primary_key=True, serialize=False
),
),
# remove old one
migrations.RemoveField('User', 'id'),
# rename to new
migrations.RenameField(
model_name='User',
old_name='uuid',
new_name='id'
),
# Update attributes
migrations.AlterField(
model_name='user',
name='id',
field=models.UUIDField(
primary_key=True, default=uuid.uuid4, serialize=False, editable=False
),
),
]
Celery
Snippet
Some necessarily need to be added to new project.
Don't have time find on the internet.
I'm too old to remember.
INIT.
gcl [email protected]:bboyadao/django_template.git proj_name \
&& cd proj_name \
&& rm -rf .git
&& git init
poetry install
SETTINGS.
ALLOWED_HOSTS = ["*"]
AUTH_APPS = [
"rest_framework_simplejwt.token_blacklist",
"oauth2_provider",
"rest_framework.authtoken",
"dj_rest_auth",
"dj_rest_auth.registration",
"allauth",
"allauth.account",
"allauth.socialaccount",
"allauth.socialaccount.providers.facebook",
"allauth.socialaccount.providers.google",
]
DOCS_APPS = ["drf_spectacular",
"drf_spectacular_sidecar"]
CELERY_APPS = ["django_celery_results",
"django_celery_beat"]
LIB_APPS = ["rest_framework",
"django_filters",
"phonenumber_field",
"corsheaders",
"fcm_django",
"actstream",
]
EXTERNAL_APPS = LIB_APPS + CELERY_APPS + AUTH_APPS + DOCS_APPS
INTERNAL_APPS = []
INSTALLED_APPS += EXTERNAL_APPS + INTERNAL_APPS