1#![no_std]
2
3use core::{ops, ptr, str};
19
20#[cfg(all(
21 target_arch = "x86_64",
22 target_feature = "avx512f",
23 target_feature = "avx512vl",
24 target_feature = "avx512ifma",
25 target_feature = "avx512bw",
26 target_feature = "avx512vbmi",
27))]
28#[path = "format_number_avx512.rs"]
29mod avx512_arch;
30
31pub(crate) const MAX_LEN: usize = 40;
32
33pub(crate) const DEC_DIGITS_LUT: &[u8; 200] = b"\
34 0001020304050607080910111213141516171819\
35 2021222324252627282930313233343536373839\
36 4041424344454647484950515253545556575859\
37 6061626364656667686970717273747576777879\
38 8081828384858687888990919293949596979899";
39
40#[derive(Copy, Clone)]
53pub struct Buffer {
54 buf: [u8; MAX_LEN],
55 start: u8,
56}
57
58impl Default for Buffer {
59 fn default() -> Self {
60 Self::new()
61 }
62}
63
64impl Buffer {
65 #[inline]
67 pub const fn new() -> Self {
68 Buffer {
69 buf: [0u8; MAX_LEN],
70 start: MAX_LEN as u8,
71 }
72 }
73
74 #[inline]
80 pub fn format<I: Integer>(&mut self, n: I) -> &str {
81 n.format_into(self);
82 self.as_str()
83 }
84
85 #[inline]
92 pub fn as_str(&self) -> &str {
93 let s = &self.buf[self.start as usize..];
94 unsafe { str::from_utf8_unchecked(s) }
95 }
96}
97
98impl ops::Deref for Buffer {
99 type Target = str;
100
101 #[inline]
102 fn deref(&self) -> &str {
103 self.as_str()
104 }
105}
106
107impl AsRef<str> for Buffer {
108 #[inline]
109 fn as_ref(&self) -> &str {
110 self.as_str()
111 }
112}
113
114impl PartialEq<&str> for Buffer {
115 #[inline]
116 fn eq(&self, other: &&str) -> bool {
117 self.as_str() == *other
118 }
119}
120
121impl PartialEq<Buffer> for Buffer {
122 #[inline]
123 fn eq(&self, other: &Buffer) -> bool {
124 self.as_str() == other.as_str()
125 }
126}
127
128impl Eq for Buffer {}
129
130#[inline]
138pub fn format_int<I: Integer>(n: I) -> Buffer {
139 let mut buf = Buffer::new();
140 buf.format(n);
141 buf
142}
143
144pub trait Integer: private::Sealed {}
152
153mod private {
154 use super::Buffer;
155
156 pub trait Sealed: Copy {
157 fn format_into(self, buf: &mut Buffer);
158 }
159}
160
161#[inline]
172fn format_u64<const FOUR_DIGIT: bool>(n: u64, buf: &mut [u8; MAX_LEN], mut pos: usize) -> usize {
173 if n < 10 {
175 pos -= 1;
176 unsafe {
177 *buf.as_mut_ptr().add(pos) = n as u8 + b'0';
178 }
179 return pos;
180 } else if n < 100 {
181 let d1 = n as usize * 2;
183 pos -= 2;
184 unsafe {
185 ptr::copy_nonoverlapping(DEC_DIGITS_LUT.as_ptr().add(d1), buf.as_mut_ptr().add(pos), 2);
186 }
187 return pos;
188 }
189
190 #[cfg(all(
191 target_arch = "x86_64",
192 target_feature = "avx512f",
193 target_feature = "avx512vl",
194 target_feature = "avx512ifma",
195 target_feature = "avx512bw",
196 target_feature = "avx512vbmi",
197 ))]
198 if n >= 100_000_000 {
199 return unsafe { avx512_arch::format_u64_avx512(n, buf, pos) };
200 }
201
202 return format_u64_scalar::<FOUR_DIGIT>(n, buf, pos);
203}
204
205#[inline]
206fn format_u64_scalar<const FOUR_DIGIT: bool>(mut n: u64, buf: &mut [u8; MAX_LEN], mut pos: usize) -> usize {
207 let buf_ptr = buf.as_mut_ptr();
208 let lut_ptr = DEC_DIGITS_LUT.as_ptr();
209
210 if FOUR_DIGIT {
211 while n >= 10000 {
212 let rem = (n % 10000) as usize;
213 n /= 10000;
214
215 let d1 = (rem / 100) * 2;
216 let d2 = (rem % 100) * 2;
217 pos -= 4;
218 unsafe {
219 ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(pos), 2);
220 ptr::copy_nonoverlapping(lut_ptr.add(d2), buf_ptr.add(pos + 2), 2);
221 }
222 }
223 }
224
225 if n >= 100 {
226 let d1 = (n % 100) as usize * 2;
227 n /= 100;
228 pos -= 2;
229 unsafe {
230 ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(pos), 2);
231 }
232 }
233
234 if n < 10 {
235 pos -= 1;
236 unsafe {
237 *buf_ptr.add(pos) = (n as u8) + b'0';
238 }
239 } else {
240 let d1 = n as usize * 2;
241 pos -= 2;
242 unsafe {
243 ptr::copy_nonoverlapping(lut_ptr.add(d1), buf_ptr.add(pos), 2);
244 }
245 }
246
247 pos
248}
249
250#[inline]
258fn format_u128(n: u128, buf: &mut [u8; MAX_LEN], mut pos: usize) -> usize {
259 if n <= u64::MAX as u128 {
262 return format_u64::<true>(n as u64, buf, pos);
263 }
264
265 let buf_ptr = buf.as_mut_ptr();
266
267 let (n, lo) = udivmod_1e19(n);
268 pos = format_u64::<true>(lo, buf, pos);
269
270 if n != 0 {
271 let target = MAX_LEN - 19;
272 unsafe {
273 ptr::write_bytes(buf_ptr.add(target), b'0', pos - target);
274 }
275 pos = target;
276
277 let (n, mid) = udivmod_1e19(n);
278 pos = format_u64::<true>(mid, buf, pos);
279
280 if n != 0 {
281 let target = MAX_LEN - 38;
282 unsafe {
283 ptr::write_bytes(buf_ptr.add(target), b'0', pos - target);
284 }
285 pos = target;
286 pos -= 1;
287 unsafe {
288 *buf_ptr.add(pos) = b'0' + n as u8;
289 }
290 }
291 }
292
293 pos
294}
295
296#[inline]
302fn udivmod_1e19(n: u128) -> (u128, u64) {
303 const D: u64 = 10_000_000_000_000_000_000;
304
305 let quot = if n < 1 << 83 {
306 ((n >> 19) as u64 / (D >> 19)) as u128
307 } else {
308 u128_mulhi(n, 156927543384667019095894735580191660403) >> 62
309 };
310
311 let rem = (n - quot * D as u128) as u64;
312 debug_assert_eq!(quot, n / D as u128);
313 debug_assert_eq!(rem as u128, n % D as u128);
314
315 (quot, rem)
316}
317
318#[inline]
322fn u128_mulhi(x: u128, y: u128) -> u128 {
323 let x_lo = x as u64;
324 let x_hi = (x >> 64) as u64;
325 let y_lo = y as u64;
326 let y_hi = (y >> 64) as u64;
327
328 let carry = (x_lo as u128 * y_lo as u128) >> 64;
329 let m = x_lo as u128 * y_hi as u128 + carry;
330 let high1 = m >> 64;
331
332 let m_lo = m as u64;
333 let high2 = (x_hi as u128 * y_lo as u128 + m_lo as u128) >> 64;
334
335 x_hi as u128 * y_hi as u128 + high1 + high2
336}
337
338macro_rules! impl_unsigned {
343 ($t:ty, $four:literal) => {
344 impl Integer for $t {}
345
346 impl private::Sealed for $t {
347 #[inline]
348 fn format_into(self, buf: &mut Buffer) {
349 buf.start = format_u64::<$four>(self as u64, &mut buf.buf, MAX_LEN) as u8;
350 }
351 }
352 };
353}
354
355macro_rules! impl_signed {
356 ($t:ty, $u:ty, $four:literal) => {
357 impl Integer for $t {}
358
359 impl private::Sealed for $t {
360 #[inline]
361 fn format_into(self, buf: &mut Buffer) {
362 let is_neg = self < 0;
363 let n: $u = if is_neg {
364 self.unsigned_abs()
366 } else {
367 self as $u
368 };
369 buf.start = format_u64::<$four>(n as u64, &mut buf.buf, MAX_LEN) as u8;
370 if is_neg {
371 buf.start -= 1;
372 buf.buf[buf.start as usize] = b'-';
373 }
374 }
375 }
376 };
377}
378
379impl_unsigned!(u8, false);
380impl_unsigned!(u16, true);
381impl_unsigned!(u32, true);
382impl_unsigned!(u64, true);
383
384impl_signed!(i8, u8, false);
385impl_signed!(i16, u16, true);
386impl_signed!(i32, u32, true);
387impl_signed!(i64, u64, true);
388
389macro_rules! impl_signed_ptr {
390 ($t:ty, $u:ty, $four:literal) => {
391 impl Integer for $t {}
392
393 impl private::Sealed for $t {
394 #[inline]
395 fn format_into(self, buf: &mut Buffer) {
396 let is_neg = self < 0;
397 let n = if is_neg {
398 self.unsigned_abs()
399 } else {
400 self as usize
401 };
402 buf.start = format_u64::<$four>(n as u64, &mut buf.buf, MAX_LEN) as u8;
403 if is_neg {
404 buf.start -= 1;
405 buf.buf[buf.start as usize] = b'-';
406 }
407 }
408 }
409 };
410}
411
412impl Integer for u128 {}
413
414impl private::Sealed for u128 {
415 #[inline]
416 fn format_into(self, buf: &mut Buffer) {
417 buf.start = format_u128(self, &mut buf.buf, MAX_LEN) as u8;
418 }
419}
420
421impl Integer for i128 {}
422
423impl private::Sealed for i128 {
424 #[inline]
425 fn format_into(self, buf: &mut Buffer) {
426 let is_neg = self < 0;
427 let n: u128 = if is_neg { self.unsigned_abs() } else { self as u128 };
428 buf.start = format_u128(n, &mut buf.buf, MAX_LEN) as u8;
429 if is_neg {
430 buf.start -= 1;
431 buf.buf[buf.start as usize] = b'-';
432 }
433 }
434}
435
436#[cfg(target_pointer_width = "16")]
437impl_unsigned!(usize, true);
438#[cfg(target_pointer_width = "32")]
439impl_unsigned!(usize, true);
440#[cfg(target_pointer_width = "64")]
441impl_unsigned!(usize, true);
442
443#[cfg(target_pointer_width = "16")]
444impl_signed_ptr!(isize, u16, true);
445#[cfg(target_pointer_width = "32")]
446impl_signed_ptr!(isize, u32, true);
447#[cfg(target_pointer_width = "64")]
448impl_signed_ptr!(isize, u64, true);
449
450#[cfg(test)]
455mod tests {
456 use super::*;
457
458 #[test]
459 fn basic_u64() {
460 let mut buf = Buffer::new();
461 assert_eq!(buf.format(0u64), "0");
462 assert_eq!(buf.format(1u64), "1");
463 assert_eq!(buf.format(9u64), "9");
464 assert_eq!(buf.format(10u64), "10");
465 assert_eq!(buf.format(42u64), "42");
466 assert_eq!(buf.format(99u64), "99");
467 assert_eq!(buf.format(100u64), "100");
468 assert_eq!(buf.format(999u64), "999");
469 assert_eq!(buf.format(1000u64), "1000");
470 assert_eq!(buf.format(9999u64), "9999");
471 assert_eq!(buf.format(10000u64), "10000");
472 assert_eq!(buf.format(12345u64), "12345");
473 assert_eq!(buf.format(65535u64), "65535");
474 assert_eq!(buf.format(u64::MAX), "18446744073709551615");
475 }
476
477 #[test]
478 fn basic_i64() {
479 let mut buf = Buffer::new();
480 assert_eq!(buf.format(0i64), "0");
481 assert_eq!(buf.format(-1i64), "-1");
482 assert_eq!(buf.format(1i64), "1");
483 assert_eq!(buf.format(-42i64), "-42");
484 assert_eq!(buf.format(i64::MAX), "9223372036854775807");
485 assert_eq!(buf.format(i64::MIN), "-9223372036854775808");
486 }
487
488 #[test]
489 fn basic_u8() {
490 let mut buf = Buffer::new();
491 assert_eq!(buf.format(0u8), "0");
492 assert_eq!(buf.format(255u8), "255");
493 assert_eq!(buf.format(u8::MAX), "255");
494 }
495
496 #[test]
497 fn basic_i8() {
498 let mut buf = Buffer::new();
499 assert_eq!(buf.format(0i8), "0");
500 assert_eq!(buf.format(127i8), "127");
501 assert_eq!(buf.format(-128i8), "-128");
502 }
503
504 #[test]
505 fn basic_u16() {
506 let mut buf = Buffer::new();
507 assert_eq!(buf.format(u16::MAX), "65535");
508 }
509
510 #[test]
511 fn basic_i16() {
512 let mut buf = Buffer::new();
513 assert_eq!(buf.format(i16::MAX), "32767");
514 assert_eq!(buf.format(i16::MIN), "-32768");
515 }
516
517 #[test]
518 fn basic_u32() {
519 let mut buf = Buffer::new();
520 assert_eq!(buf.format(u32::MAX), "4294967295");
521 }
522
523 #[test]
524 fn basic_i32() {
525 let mut buf = Buffer::new();
526 assert_eq!(buf.format(i32::MAX), "2147483647");
527 assert_eq!(buf.format(i32::MIN), "-2147483648");
528 }
529
530 #[test]
531 fn basic_u128() {
532 let mut buf = Buffer::new();
533 assert_eq!(buf.format(0u128), "0");
534 assert_eq!(buf.format(1u128), "1");
535 assert_eq!(buf.format(10u128), "10");
536 assert_eq!(buf.format(100u128), "100");
537 assert_eq!(buf.format(1000u128), "1000");
538 assert_eq!(buf.format(10000u128), "10000");
539 assert_eq!(buf.format(u128::MAX), "340282366920938463463374607431768211455");
540 }
541
542 #[test]
543 fn basic_i128() {
544 let mut buf = Buffer::new();
545 assert_eq!(buf.format(0i128), "0");
546 assert_eq!(buf.format(-1i128), "-1");
547 assert_eq!(buf.format(i128::MAX), "170141183460469231731687303715884105727");
548 assert_eq!(buf.format(i128::MIN), "-170141183460469231731687303715884105728");
549 }
550
551 #[test]
552 fn powers_of_ten() {
553 let mut buf = Buffer::new();
554 assert_eq!(buf.format(1u64), "1");
555 assert_eq!(buf.format(10u64), "10");
556 assert_eq!(buf.format(100u64), "100");
557 assert_eq!(buf.format(1000u64), "1000");
558 assert_eq!(buf.format(10000u64), "10000");
559 assert_eq!(buf.format(100000u64), "100000");
560 assert_eq!(buf.format(1000000u64), "1000000");
561 assert_eq!(buf.format(10000000u64), "10000000");
562 assert_eq!(buf.format(100000000u64), "100000000");
563 assert_eq!(buf.format(1000000000u64), "1000000000");
564 assert_eq!(buf.format(10000000000u64), "10000000000");
565 assert_eq!(buf.format(100000000000u64), "100000000000");
566 assert_eq!(buf.format(1000000000000u64), "1000000000000");
567 assert_eq!(buf.format(10000000000000u64), "10000000000000");
568 assert_eq!(buf.format(100000000000000u64), "100000000000000");
569 assert_eq!(buf.format(1000000000000000u64), "1000000000000000");
570 assert_eq!(buf.format(10000000000000000u64), "10000000000000000");
571 assert_eq!(buf.format(100000000000000000u64), "100000000000000000");
572 assert_eq!(buf.format(1000000000000000000u64), "1000000000000000000");
573 assert_eq!(buf.format(10000000000000000000u64), "10000000000000000000");
574 }
575
576 #[test]
577 fn u128_edge_cases() {
578 let mut buf = Buffer::new();
579
580 assert_eq!(buf.format(9999999999999999999u128), "9999999999999999999");
582
583 assert_eq!(buf.format(10000000000000000000u128), "10000000000000000000");
585
586 assert_eq!(buf.format(10000000000000000001u128), "10000000000000000001");
588
589 assert_eq!(buf.format(99999999999999999999u128), "99999999999999999999");
591
592 assert_eq!(buf.format(100000000000000000000u128), "100000000000000000000");
594
595 assert_eq!(
597 buf.format(99999999999999999999999999999999999999u128),
598 "99999999999999999999999999999999999999"
599 );
600 }
601
602 #[test]
603 fn u128_large_values() {
604 let mut buf = Buffer::new();
605
606 assert_eq!(buf.format(1u128 << 83), "9671406556917033397649408");
608 assert_eq!(buf.format((1u128 << 83) + 1), "9671406556917033397649409");
609 assert_eq!(buf.format((1u128 << 83) - 1), "9671406556917033397649407");
610 assert_eq!(buf.format(1u128 << 84), "19342813113834066795298816");
611
612 let e25 = 10_000_000_000_000_000_000_000_000u128;
614 assert_eq!(buf.format(e25), "10000000000000000000000000");
615 assert_eq!(buf.format(e25 - 1), "9999999999999999999999999");
616 assert_eq!(buf.format(e25 + 1), "10000000000000000000000001");
617
618 let e30: u128 = 1_000_000_000_000_000_000_000_000_000_000;
620 assert_eq!(buf.format(e30), "1000000000000000000000000000000");
621 assert_eq!(buf.format(e30 - 1), "999999999999999999999999999999");
622 }
623
624 #[test]
625 fn oneshot() {
626 assert!(format_int(42u64) == "42");
627 assert!(format_int(-99i32) == "-99");
628 assert!(format_int(0u8) == "0");
629 assert!(format_int(255u8) == "255");
630 }
631
632 #[test]
633 fn deref_to_str() {
634 let mut buf = Buffer::new();
635 buf.format(123u64);
636 let s: &str = &buf;
637 assert_eq!(s, "123");
638 }
639
640 #[test]
641 fn as_ref() {
642 let mut buf = Buffer::new();
643 buf.format(456u64);
644 assert_eq!(buf.as_ref(), "456");
645 }
646
647 #[test]
648 fn partial_eq_str() {
649 let mut buf = Buffer::new();
650 buf.format(789u64);
651 assert_eq!(buf.as_str(), "789");
652 assert_ne!(buf.as_str(), "000");
653 }
654
655 #[test]
656 fn partial_eq_buffer() {
657 let mut a = Buffer::new();
658 let mut b = Buffer::new();
659 a.format(42u64);
660 b.format(42u64);
661 assert_eq!(a.as_str(), b.as_str());
662 b.format(99u64);
663 assert_ne!(a.as_str(), b.as_str());
664 }
665
666 #[test]
667 fn clone() {
668 let mut a = Buffer::new();
669 a.format(42u64);
670 let b = a.clone();
671 assert_eq!(a.as_str(), b.as_str());
672 }
673
674 #[test]
675 fn const_new() {
676 const BUF: Buffer = Buffer::new();
677 assert_eq!(BUF.as_str(), "");
678 }
679
680 #[test]
681 fn reuse() {
682 let mut buf = Buffer::new();
683 assert_eq!(buf.format(1u64), "1");
684 assert_eq!(buf.format(12u64), "12");
685 assert_eq!(buf.format(123u64), "123");
686 assert_eq!(buf.format(1234u64), "1234");
687 assert_eq!(buf.format(12345u64), "12345");
688 }
689}
690
691#[cfg(all(
692 test,
693 target_arch = "x86_64",
694 target_feature = "avx512f",
695 target_feature = "avx512vl",
696 target_feature = "avx512ifma",
697 target_feature = "avx512bw",
698 target_feature = "avx512vbmi",
699))]
700mod avx512_tests {
701 extern crate alloc;
702 use alloc::{borrow::ToOwned, string::String};
703
704 use super::*;
705
706 fn format_avx512(n: u64) -> String {
707 let mut buf = [0u8; MAX_LEN];
708 let pos = unsafe { avx512_arch::format_u64_avx512(n, &mut buf, MAX_LEN) };
709 let s = &buf[pos..MAX_LEN];
710 core::str::from_utf8(s).unwrap().to_owned()
711 }
712
713 fn format_scalar(n: u64) -> String {
714 let mut buf = [0u8; MAX_LEN];
715 let pos = format_u64_scalar::<true>(n, &mut buf, MAX_LEN);
716 let s = &buf[pos..MAX_LEN];
717 core::str::from_utf8(s).unwrap().to_owned()
718 }
719
720 #[test]
721 fn boundaries() {
722 let cases = [
724 100_000_000,
725 999_999_999,
726 1_000_000_000,
727 9_999_999_999,
728 10_000_000_000,
729 99_999_999_999,
730 100_000_000_000,
731 999_999_999_999,
732 1_000_000_000_000,
733 9_999_999_999_999,
734 10_000_000_000_000,
735 99_999_999_999_999,
736 100_000_000_000_000,
737 999_999_999_999_999,
738 1_000_000_000_000_000,
739 9_999_999_999_999_999,
740 10_000_000_000_000_000,
741 99_999_999_999_999_999,
742 100_000_000_000_000_000,
743 999_999_999_999_999_999,
744 1_000_000_000_000_000_000,
745 9_999_999_999_999_999_999,
746 10_000_000_000_000_000_000,
747 u64::MAX,
748 ];
749 for &n in &cases {
750 let avx = format_avx512(n);
751 let scalar = format_scalar(n);
752 assert_eq!(avx, scalar, "mismatch at n={}", n);
753 }
754 }
755
756 #[test]
757 fn powers_and_neighbors() {
758 let mut base = 100_000_000u64;
760 for _ in 8..=19 {
761 for &delta in &[0u64, 1, 2, 10, base - 1, base - 2] {
762 let n = base.saturating_sub(delta);
763 if n >= 100_000_000 {
764 assert_eq!(format_avx512(n), format_scalar(n), "mismatch at n={}", n);
765 }
766 }
767 if let Some(next) = base.checked_mul(10) {
768 base = next;
769 } else {
770 break;
771 }
772 }
773 }
774
775 #[test]
776 fn random() {
777 use rand::{RngExt, SeedableRng, rngs::SmallRng};
778 let mut rng = SmallRng::seed_from_u64(0xdeadbeef);
779 for _ in 0..10_000 {
780 let n = rng.random_range(100_000_000u64..=u64::MAX);
781 assert_eq!(format_avx512(n), format_scalar(n), "mismatch at n={}", n);
782 }
783 }
784}